Update Beats to ECS 1.8.0 (#23465)

Incorporates ECS 1.8 changes from the following PRs: Support host.type field in add_host_metadata processor and Auditbeat's system/host #23513 Winlogbeat #23563 Auditbeat auditd #23594 Journalbeat #23737 Packetbeat #23783 Filebeat: auditd #23723 cisco #23819 cef #23832 crowdstrike falcon #23875 fortinet firewall #23902 microsoft #23897 elasticsearch/audit #24000 Gsuite/Workspace #23709 o365 #23896 zoom #23904 okta #23929 aws/cloudtrail #23911 aws/s3access #23920 azure #23927 juniper/srx #23936 panw #23931 sophos/xg #23967 system/auth #23961 mysqlenterprise #23978 zeek #23847 Make all Beats and modules report ECS 1.8.0 #23992 Closes #23118 Co-authored-by: Marc Guasch <[email protected]>
elastic · Feb 16, 2021 · 048c3cc · 048c3cc
1 parent 239cff7
commit 048c3cc
Show file tree

Hide file tree

Showing 510 changed files with 26,495 additions and 3,814 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -381,6 +381,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix httpjson input logging so it doesn't conflict with ECS. {pull}23972[23972]
 - Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
 - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709]
+- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
+- Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
 
 *Heartbeat*
 
@@ -604,6 +606,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629]
 - Added new decode_xml processor to libbeat that is available to all beat types. {pull}23678[23678]
 - Add deployment name in pod's meta. {pull}23610[23610]
+- Added ECS 1.8 `host.os.type` field to `add_host_metadata` processor. {pull}23513[23513]
 - Add `selector` information in kubernetes services' metadata. {pull}23730[23730]
 
 *Auditbeat*
@@ -625,6 +628,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647]
 - Add ECS 1.7 `configuration` categorization in certain events in auditd module. {pull}23000[23000]
 - Improve file_integrity monitoring when a file is created/deleted in quick succession. {issue}17347[17347] {pull}22170[22170]
+- system/host: Add new ECS 1.8 field `os.type` in `host.os.type`. {pull}23513[23513]
+- Update Auditbeat auditd module to ECS 1.8 {pull}23594[23594] {issue}23118[23118]
 
 *Filebeat*
 
@@ -835,6 +840,26 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added RFC6587 framing option for tcp and unix inputs {issue}23663[23663] {pull}23724[23724]
 - Added string splitting for httpjson input {pull}24022[24022]
 - Added field mappings for Netflow/IPFIX vendor fields that are known to Filebeat. {issue}23771[23771]
+- Upgrade Cisco ASA/FTD/Umbrella to ECS 1.8.0. {pull}23819[23819]
+- Add new ECS user and categories features to google_workspace/gsuite {issue}23118[23118] {pull}23709[23709]
+- Move crowdstrike JS processor to ingest pipelines and upgrade to ECS 1.8.0 {issue}23118[23118] {pull}23875[23875]
+- Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
+- Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
+- Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
+- Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
+- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
+- Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
+- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
+- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
+- Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
+- Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
+- Upgrade juniper/srx to ecs 1.8.0. {issue}23118[23118] {pull}23936[23936]
+- Update mysqlenterprise module to ECS 1.8. {issue}23118[23118] {pull}23978[23978]
+- Upgrade sophos/xg fileset to ECS 1.8.0. {issue}23118[23118] {pull}23967[23967]
+- Upgrade system/auth to ECS 1.8 {issue}23118[23118] {pull}23961[23961]
+- Upgrade elasticsearch/audit to ECS 1.8 {issue}23118[23118] {pull}24000[24000]
+- Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline {issue}23118[23118] {pull}23929[23929]
+- Update zoom module to ECS 1.8. {pull}23904[23904] {issue}23118[23118]
 
 *Heartbeat*
 
@@ -843,6 +868,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Journalbeat*
 
+- Update Journalbeat to ECS 1.8. {pull}23737[23737]
 
 *Metricbeat*
 
@@ -978,6 +1004,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change build process for x-pack distribution {pull}21979[21979]
 - Tuned the internal queue size to reduce the chances of events being dropped. {pull}22650[22650]
 - Add support for "http.request.mime_type" and "http.response.mime_type". {pull}22940[22940]
+- Upgrade to ECS 1.8.0. {pull}23783[23783]
 
 *Functionbeat*
 
@@ -1004,6 +1031,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
 - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
 - Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684]
+- Add new ECS 1.8 improvements. {pull}23563[23563]
 
 *Elastic Log Driver*
 
@@ -1038,7 +1066,3 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Known Issue
 
 *Journalbeat*
-
-
-
-
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -5891,11 +5891,11 @@ This Agreement is governed by the laws of the State of New York and the intellec
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.6.0
+Version: v1.0.0-beta2.0.20210202203518-638aa2bb5271
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.6.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.0.0-beta2.0.20210202203518-638aa2bb5271/LICENSE.txt:
 
 
                                  Apache License
@@ -6547,11 +6547,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-libaudit/v2
-Version: v2.1.0
+Version: v2.2.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.1.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.2.0/LICENSE.txt:
 
 
                                  Apache License
@@ -7665,11 +7665,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-sysinfo
-Version: v1.3.0
+Version: v1.5.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.3.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.5.0/LICENSE.txt:
 
 
                                  Apache License

diff --git a/auditbeat/_meta/fields.common.yml b/auditbeat/_meta/fields.common.yml
@@ -66,27 +66,6 @@
         type: keyword
         description: Audit user name.
 
-    - name: effective
-      type: group
-      description: Effective user information.
-      fields:
-      - name: id
-        type: keyword
-        description: Effective user ID.
-      - name: name
-        type: keyword
-        description: Effective user name.
-      - name: group
-        type: group
-        description: Effective group information.
-        fields:
-        - name: id
-          type: keyword
-          description: Effective group ID.
-        - name: name
-          type: keyword
-          description: Effective group name.
-
     - name: filesystem
       type: group
       description: Filesystem user information.

diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go
@@ -35,7 +35,7 @@ const (
 	Name = "auditbeat"
 
 	// ecsVersion specifies the version of ECS that Auditbeat is implementing.
-	ecsVersion = "1.7.0"
+	ecsVersion = "1.8.0"
 )
 
 // RootCmd for running auditbeat.