From 8ec579483341d935fc984d984ea95c892f14e57d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 15:42:40 -0400 Subject: [PATCH 01/20] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f621b256..7d52aac7f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.0-foxtrot From 1df19faf5c3ea45a94e367718c0cd69d12a9e1ed Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 15:44:50 -0400 Subject: [PATCH 02/20] Elastic 8.14.3 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 8a6effa5c8..902aabaa3f 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.2" +ELASTIC_AGENT_TARBALL_VERSION="8.14.3" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From fbd0dbd048d9e92455493cacbaffaeec7a80e206 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 15:46:55 -0400 Subject: [PATCH 03/20] Elastic 8.14.3 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index fc0896009e..8177adb5c2 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 678b232c24d03d6bfe943293e231e3e60d30d5ec Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 15:48:01 -0400 Subject: [PATCH 04/20] Elastic 8.14.3 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 50e2ba45c1..e4935b9594 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.3","id": "8.14.3","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From f051ddc7f021f248b9a55e85805120d4c2b27f45 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 17 Jul 2024 09:50:26 -0400 Subject: [PATCH 05/20] Remove pipelines --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 656e9b3d9f..0eec04c22a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -385,7 +385,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 - [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 + [[ "$INSTALLEDVERSION" == 2.4.80 ]] && true } @@ -660,6 +660,7 @@ up_to_2.4.80() { up_to_2.4.90() { echo "Nothing to apply" INSTALLEDVERSION=2.4.90 + rm /opt/so/state/espipelines.txt } add_detection_test_pillars() { From c0bb395571ef93a936ede467d9b569d239b5d888 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 17 Jul 2024 09:51:51 -0400 Subject: [PATCH 06/20] Remove pipeline file removal --- salt/manager/tools/sbin/soup | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0eec04c22a..1f8458aa4c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -660,7 +660,6 @@ up_to_2.4.80() { up_to_2.4.90() { echo "Nothing to apply" INSTALLEDVERSION=2.4.90 - rm /opt/so/state/espipelines.txt } add_detection_test_pillars() { From 2d0de875302b5c5d2560efa31c3af9d583a0b693 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 17 Jul 2024 15:19:46 +0000 Subject: [PATCH 07/20] Add component templates for Fleet metrics --- ...ics-fleet_server.agent_status@package.json | 201 ++++++++++++++++++ ...s-fleet_server.agent_versions@package.json | 102 +++++++++ 2 files changed, 303 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json create mode 100644 salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json new file mode 100644 index 0000000000..8fc83f9cb8 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json @@ -0,0 +1,201 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_status-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agents": { + "properties": { + "offline": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "total": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "updating": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "inactive": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "healthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unenrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "enrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy_reason": { + "properties": { + "output": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "input": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "other": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + }, + "upgrading_step": { + "properties": { + "rollback": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "requested": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "restarting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "downloading": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "scheduled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "extracting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "replacing": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "failed": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "watching": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json new file mode 100644 index 0000000000..af3323ee90 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json @@ -0,0 +1,102 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agent": { + "properties": { + "count": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "version": { + "time_series_dimension": true, + "type": "keyword" + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} From f78a5d1a780b80d74d933579da012c42fec7c45b Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 17 Jul 2024 15:42:40 +0000 Subject: [PATCH 08/20] Remove pipeline file --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1f8458aa4c..0311e48a4e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -486,6 +486,7 @@ post_to_2.4.80() { post_to_2.4.90() { echo "Nothing to apply" + rm /opt/so/state/espipelines.txt POSTVERSION=2.4.90 } From 612716ee69e675eff5f8dee243543c04dba13ba6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 17 Jul 2024 17:35:41 +0000 Subject: [PATCH 09/20] Apply ES to load pipelines --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0311e48a4e..5a293afe94 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -487,6 +487,7 @@ post_to_2.4.80() { post_to_2.4.90() { echo "Nothing to apply" rm /opt/so/state/espipelines.txt + salt-call state.apply elasticsearch POSTVERSION=2.4.90 } From bdba621442ea6c1c3db1d3824f89d7a273025821 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 16:32:28 +0000 Subject: [PATCH 10/20] Remove soup changes --- salt/manager/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5a293afe94..1f8458aa4c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -486,8 +486,6 @@ post_to_2.4.80() { post_to_2.4.90() { echo "Nothing to apply" - rm /opt/so/state/espipelines.txt - salt-call state.apply elasticsearch POSTVERSION=2.4.90 } From dd852497815b40983c1b3a273c148ef7c239ce39 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 16:36:41 +0000 Subject: [PATCH 11/20] Remove Fleet final pipeline --- salt/elasticsearch/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 27a8a0fd66..4253b1d006 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,13 @@ esingestconf: - user: 930 - group: 939 +# Remove .fleet_final_pipeline-1 because we are using global@custom now +so-fleet-final-pipeline-remove: + file.absent: + - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1 + - onchanges: + - file: esingestconf + # Auto-generate Elasticsearch ingest node pipelines from pillar {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} es_ingest_conf_{{pipeline}}: From 6f44d39b18df848bbc760758125ab463ff121f5c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 16:37:03 +0000 Subject: [PATCH 12/20] Remove Fleet final pipeline file --- .../files/ingest/.fleet_final_pipeline-1 | 107 ------------------ 1 file changed, 107 deletions(-) delete mode 100644 salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 deleted file mode 100644 index 233cd647b4..0000000000 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ /dev/null @@ -1,107 +0,0 @@ -{ - "version": 3, - "_meta": { - "managed_by": "fleet", - "managed": true - }, - "description": "Final pipeline for processing all incoming Fleet Agent documents. \n", - "processors": [ - { - "date": { - "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)", - "tag": "truncate-subseconds-event-ingested", - "field": "_ingest.timestamp", - "target_field": "event.ingested", - "formats": [ - "ISO8601" - ], - "output_format": "date_time_no_millis", - "ignore_failure": true - } - }, - { - "remove": { - "description": "Remove any pre-existing untrusted values.", - "field": [ - "event.agent_id_status", - "_security" - ], - "ignore_missing": true - } - }, - { - "set_security_user": { - "field": "_security", - "properties": [ - "authentication_type", - "username", - "realm", - "api_key" - ] - } - }, - { - "script": { - "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n", - "tag": "agent-id-status", - "source": "boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);", - "params": { - "trusted_users": [ - { - "username": "elastic/fleet-server", - "realm": "_service_account" - }, - { - "username": "cloud-internal-agent-server", - "realm": "found" - }, - { - "username": "elastic", - "realm": "reserved" - } - ] - } - } - }, - { - "remove": { - "field": "_security", - "ignore_missing": true - } - }, - { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, - { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, - { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, - { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, - { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, - { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } - ], - "on_failure": [ - { - "remove": { - "field": "_security", - "ignore_missing": true, - "ignore_failure": true - } - }, - { - "append": { - "field": "error.message", - "value": [ - "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}" - ] - } - } - ] -} From e789c17bc3ba774751eb18c7ed0734bd2f045dc7 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 16:37:37 +0000 Subject: [PATCH 13/20] Add global@custom pipeline file --- salt/elasticsearch/files/ingest/global@custom | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/global@custom diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom new file mode 100644 index 0000000000..dbf215fb1b --- /dev/null +++ b/salt/elasticsearch/files/ingest/global@custom @@ -0,0 +1,27 @@ +{ + "version": 3, + "_meta": { + "managed_by": "securityonion", + "managed": true + }, + "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", + "processors": [ + { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, + { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, + { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, + { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, + { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, + { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, + { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, + { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } + ] +} From 17f37750e528fc6192fe9d1f8bc40129fbdc91f6 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 16:46:18 +0000 Subject: [PATCH 14/20] Remove onchanges condition --- salt/elasticsearch/config.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 4253b1d006..a3dd189adf 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -122,8 +122,6 @@ esingestconf: so-fleet-final-pipeline-remove: file.absent: - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1 - - onchanges: - - file: esingestconf # Auto-generate Elasticsearch ingest node pipelines from pillar {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} From c55fa6dc6af868f905c7a9e1b13cdb7fec47af17 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 23 Jul 2024 17:48:32 +0000 Subject: [PATCH 15/20] Fix pattern for pipelines --- salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 71c40c1ca2..b76a0e0f01 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." - for i in .[a-z]* *; + for i in *; do echo $i; retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i" From 8538f2eca2bea208c9ae02f2c27eda67c21cd03e Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Jul 2024 09:40:30 -0400 Subject: [PATCH 16/20] Elastic Agent update --- salt/manager/tools/sbin/soup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1f8458aa4c..7facde9a6c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -658,8 +658,9 @@ up_to_2.4.80() { } up_to_2.4.90() { - echo "Nothing to apply" - INSTALLEDVERSION=2.4.90 + # Elastic Update for this release, so download Elastic Agent files + determine_elastic_agent_upgrade + INSTALLEDVERSION=2.4.30 } add_detection_test_pillars() { From f2ad4c40e6506d2909570a7ab05e6991b6c471c0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Jul 2024 10:38:05 -0400 Subject: [PATCH 17/20] Fix update for 2.4.90 --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 7facde9a6c..13b8605e0e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -385,7 +385,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 - [[ "$INSTALLEDVERSION" == 2.4.80 ]] && + [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 true } @@ -660,7 +660,7 @@ up_to_2.4.80() { up_to_2.4.90() { # Elastic Update for this release, so download Elastic Agent files determine_elastic_agent_upgrade - INSTALLEDVERSION=2.4.30 + INSTALLEDVERSION=2.4.90 } add_detection_test_pillars() { From 5cba4d7d9b4d77664b1ca01846657b2d25694d0e Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 29 Jul 2024 13:16:14 -0400 Subject: [PATCH 18/20] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f9..fd912cb251 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.100 From c753a7cffa6cfd1f3b39635a3e4e3b0aeb351318 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 29 Jul 2024 13:18:07 -0400 Subject: [PATCH 19/20] Add function for 2.4.100 --- salt/manager/tools/sbin/soup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 13b8605e0e..ecdd30fe5c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -386,6 +386,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 + [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.100 true } @@ -658,9 +659,12 @@ up_to_2.4.80() { } up_to_2.4.90() { + INSTALLEDVERSION=2.4.90 +} +up_to_2.4.100() { # Elastic Update for this release, so download Elastic Agent files determine_elastic_agent_upgrade - INSTALLEDVERSION=2.4.90 + INSTALLEDVERSION=2.4.100 } add_detection_test_pillars() { From 6d008546f198472224223a31a265517fab31c399 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 30 Jul 2024 09:26:46 -0400 Subject: [PATCH 20/20] Fix pre and add post for 2.4.100 --- salt/manager/tools/sbin/soup | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 13ee8812af..221eb37992 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -401,7 +401,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 - [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.100 + [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100 true } @@ -421,6 +421,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 + [[ "$POSTVERSION" == 2.4.90 ]] && post_to_2.4.100 true } @@ -505,6 +506,11 @@ post_to_2.4.90() { POSTVERSION=2.4.90 } +post_to_2.4.100() { + echo "Nothing to apply" + POSTVERSION=2.4.100 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."