[libbeat] Add wineventlog schema to decode_xml processor

CHANGELOG.next.asciidoc

@@ -597,6 +597,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added "add_network_direction" processor for determining perimeter-based network direction. {pull}23076[23076]
 - Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883]
 - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
+- Add `wineventlog` schema to `decode_xml` processor. {issue}23910[23910] {pull}24726[24726]
 
 *Auditbeat*
 

libbeat/docs/processors-list.asciidoc

@@ -116,6 +116,9 @@ endif::[]
 ifndef::no_urldecode_processor[]
 * <<urldecode, `urldecode`>>
 endif::[]
+ifndef::no_decode_xml_processor[]
+* <<decode_xml, `decode_xml`>>
+endif::[]
 //# end::processors-list[]
 
 //# tag::processors-include[]

libbeat/processors/decode_xml/config.go

@@ -25,6 +25,7 @@ type decodeXMLConfig struct {
 	ToLower       bool    `config:"to_lower"`
 	IgnoreMissing bool    `config:"ignore_missing"`
 	IgnoreFailure bool    `config:"ignore_failure"`
+	Schema        string  `config:"schema"`
 }
 
 func defaultConfig() decodeXMLConfig {

libbeat/processors/decode_xml/decode_xml.go

@@ -21,12 +21,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
-	"github.com/elastic/beats/v7/libbeat/common/encoding/xml"
 	"github.com/elastic/beats/v7/libbeat/common/jsontransform"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/processors"
@@ -36,7 +34,9 @@ import (
 
 type decodeXML struct {
 	decodeXMLConfig
-	log *logp.Logger
+
+	decode decoder
+	log    *logp.Logger
 }
 
 var (
@@ -51,9 +51,15 @@ const (
 func init() {
 	processors.RegisterPlugin(procName,
 		checks.ConfigChecked(New,
-			checks.RequireFields("fields"),
-			checks.AllowedFields("fields", "overwrite_keys", "add_error_key", "target", "document_id")))
+			checks.RequireFields("field"),
+			checks.AllowedFields(
+				"field", "target_field",
+				"overwrite_keys", "document_id",
+				"to_lower", "ignore_missing",
+				"ignore_failure", "schema",
+			)))
 	jsprocessor.RegisterPlugin(procName, New)
+	registerDecoders()
 }
 
 // New constructs a new decode_xml processor.
@@ -77,6 +83,7 @@ func newDecodeXML(config decodeXMLConfig) (processors.Processor, error) {
 
 	return &decodeXML{
 		decodeXMLConfig: config,
+		decode:          newDecoder(config),
 		log:             logp.NewLogger(logName),
 	}, nil
 }
@@ -104,9 +111,9 @@ func (x *decodeXML) run(event *beat.Event) error {
 		return errFieldIsNotString
 	}
 
-	xmlOutput, err := x.decodeField(text)
+	xmlOutput, err := x.decode([]byte(text))
 	if err != nil {
-		return err
+		return fmt.Errorf("error decoding XML field: %w", err)
 	}
 
 	var id string
@@ -131,19 +138,6 @@ func (x *decodeXML) run(event *beat.Event) error {
 	return nil
 }
 
-func (x *decodeXML) decodeField(data string) (decodedData map[string]interface{}, err error) {
-	dec := xml.NewDecoder(strings.NewReader(data))
-	if x.ToLower {
-		dec.LowercaseKeys()
-	}
-
-	out, err := dec.Decode()
-	if err != nil {
-		return nil, fmt.Errorf("error decoding XML field: %w", err)
-	}
-	return out, nil
-}
-
 func (x *decodeXML) String() string {
 	json, _ := json.Marshal(x.decodeXMLConfig)
 	return procName + "=" + string(json)

libbeat/processors/decode_xml/decode_xml_test.go

@@ -57,7 +57,7 @@ func TestDecodeXML(t *testing.T) {
 				</catalog>`,
 			},
 			Output: common.MapStr{
-				"xml": map[string]interface{}{
+				"xml": common.MapStr{
 					"catalog": map[string]interface{}{
 						"book": map[string]interface{}{
 							"author": "William H. Gaddis",
@@ -125,7 +125,7 @@ func TestDecodeXML(t *testing.T) {
 				</catalog>`,
 			},
 			Output: common.MapStr{
-				"message": map[string]interface{}{
+				"message": common.MapStr{
 					"catalog": map[string]interface{}{
 						"book": map[string]interface{}{
 							"author": "William H. Gaddis",
@@ -158,7 +158,7 @@ func TestDecodeXML(t *testing.T) {
 				</catalog>`,
 			},
 			Output: common.MapStr{
-				"message": map[string]interface{}{
+				"message": common.MapStr{
 					"catalog": map[string]interface{}{
 						"book": []interface{}{
 							map[string]interface{}{
@@ -203,7 +203,7 @@ func TestDecodeXML(t *testing.T) {
 				</catalog>`,
 			},
 			Output: common.MapStr{
-				"message": map[string]interface{}{
+				"message": common.MapStr{
 					"catalog": map[string]interface{}{
 						"book": []interface{}{
 							map[string]interface{}{
@@ -448,7 +448,7 @@ func TestXMLToDocumentID(t *testing.T) {
 	require.NoError(t, err)
 
 	wantFields := common.MapStr{
-		"message": map[string]interface{}{
+		"message": common.MapStr{
 			"catalog": map[string]interface{}{
 				"book": map[string]interface{}{
 					"author": "William H. Gaddis",

libbeat/processors/decode_xml/docs/decode_xml.asciidoc

@@ -26,7 +26,7 @@ processors:
 By default any decoding errors that occur will stop the processing chain and the
 error will be added to `error.message` field. To ignore all errors and continue
 to the next processor you can set `ignore_failure: true`. To specifically
-ignore failures caused by `field` not existing use `ignore_missing`.
+ignore failures caused by `field` not existing you can set `ignore_missing: true`.
 
 [source,yaml]
 -------
@@ -55,15 +55,13 @@ Example XML input:
 
 [source,xml]
 -------------------------------------------------------------------------------
-{
-  <catalog>
-    <book seq="1">
-      <author>William H. Gaddis</author>
-      <title>The Recognitions</title>
-      <review>One of the great seminal American novels of the 20th century.</review>
-    </book>
-  </catalog>
-}
+<catalog>
+  <book seq="1">
+    <author>William H. Gaddis</author>
+    <title>The Recognitions</title>
+    <review>One of the great seminal American novels of the 20th century.</review>
+  </book>
+</catalog>
 -------------------------------------------------------------------------------
 
 Will produce the following output:
@@ -97,10 +95,13 @@ value (`target_field:`) is treated as if the field was not set at all.
 
 `overwrite_keys`:: (Optional) A boolean that specifies whether keys that already
 exist in the event are overwritten by keys from the decoded XML object. The
-default value is false.
+default value is `true`.
 
 `to_lower`:: (Optional) Converts all keys to lowercase. Accepts either true or
-false. The default value is true.
+false. The default value is `true`.
+
+`schema`:: (Optional) Specifies the schema of the message. Accepted schemas: `wineventlog`.
+If no schema is specified it defaults to using the regular XML to JSON conversion.
 
 `document_id`:: (Optional) XML key to use as the document ID. If configured, the
 field will be removed from the original XML document and stored in
@@ -113,3 +114,80 @@ when a specified field does not exist. Defaults to `false`.
 Defaults to `false`.
 
 See <<conditions>> for a list of supported conditions.
+
+
+==== Schemas
+
+When a schema is defined, the specific decoder will parse the configured field.
+The ouput of the parsing will be specific to that schema.
+
+===== Wineventlog
+
+The `wineventlog` schema decodes Windows Events.
+
+The decoder will always output the fields formatted in the same way, the
+`to_lower` option will be ignored when using this schema decoder.
+The output fields will be the same as the
+{winlogbeat-ref}/exported-fields-winlog.html#_winlog[winlogbeat winlog fields].
+
+Example:
+
+[source,yaml]
+-------------------------------------------------------------------------------
+processors:
+  - decode_xml:
+      field: event.original
+      target_field: winlog
+      to_lower: false
+-------------------------------------------------------------------------------
+
+[source,json]
+-------------------------------------------------------------------------------
+{
+  "event": {
+    "original": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-a5ba-3e3b0328c30d}'/><EventID>4672</EventID><Version>0</Version><Level>0</Level><Task>12548</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2021-03-23T09:56:13.137310000Z'/><EventRecordID>11303</EventRecordID><Correlation ActivityID='{ffb23523-1f32-0000-c335-b2ff321fd701}'/><Execution ProcessID='652' ThreadID='4660'/><Channel>Security</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>SYSTEM</Data><Data Name='SubjectDomainName'>NT AUTHORITY</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='PrivilegeList'>SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Data></EventData><RenderingInfo Culture='en-US'><Message>Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Message><Level>Information</Level><Task>Special Logon</Task><Opcode>Info</Opcode><Channel>Security</Channel><Provider>Microsoft Windows security auditing.</Provider><Keywords><Keyword>Audit Success</Keyword></Keywords></RenderingInfo></Event>"
+  }
+}
+-------------------------------------------------------------------------------
+
+Will produce the following output:
+
+[source,json]
+-------------------------------------------------------------------------------
+{
+  "event": {
+    "original": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-a5ba-3e3b0328c30d}'/><EventID>4672</EventID><Version>0</Version><Level>0</Level><Task>12548</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime='2021-03-23T09:56:13.137310000Z'/><EventRecordID>11303</EventRecordID><Correlation ActivityID='{ffb23523-1f32-0000-c335-b2ff321fd701}'/><Execution ProcessID='652' ThreadID='4660'/><Channel>Security</Channel><Computer>vagrant</Computer><Security/></System><EventData><Data Name='SubjectUserSid'>S-1-5-18</Data><Data Name='SubjectUserName'>SYSTEM</Data><Data Name='SubjectDomainName'>NT AUTHORITY</Data><Data Name='SubjectLogonId'>0x3e7</Data><Data Name='PrivilegeList'>SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Data></EventData><RenderingInfo Culture='en-US'><Message>Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege</Message><Level>Information</Level><Task>Special Logon</Task><Opcode>Info</Opcode><Channel>Security</Channel><Provider>Microsoft Windows security auditing.</Provider><Keywords><Keyword>Audit Success</Keyword></Keywords></RenderingInfo></Event>"
+  },
+  "winlog": {
+    "channel": "Security",
+    "outcome": "success",
+    "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}",
+    "level": "information",
+    "event_id": 4672,
+    "provider_name": "Microsoft-Windows-Security-Auditing",
+    "record_id": 11303,
+    "computer_name": "vagrant",
+    "keywords_raw": 9232379236109516800,
+    "opcode": "Info",
+    "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}",
+    "event_data": {
+      "SubjectUserSid": "S-1-5-18",
+      "SubjectUserName": "SYSTEM",
+      "SubjectDomainName": "NT AUTHORITY",
+      "SubjectLogonId": "0x3e7",
+      "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege"
+    },
+    "task": "Special Logon",
+    "keywords": [
+      "Audit Success"
+    ],
+    "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege",
+    "process": {
+      "pid": 652,
+      "thread": {
+        "id": 4660
+      }
+    }
+  }
+}
+-------------------------------------------------------------------------------

libbeat/processors/decode_xml/schema.go

@@ -0,0 +1,95 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package decode_xml
+
+import (
+	"bytes"
+	"errors"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/encoding/xml"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/winlogbeat/sys/winevent"
+)
+
+const wineventlogSchema = "wineventlog"
+
+type newDecoderFunc func(cfg decodeXMLConfig) decoder
+type decoder func(p []byte) (common.MapStr, error)
+
+var (
+	registeredDecoders                = map[string]newDecoderFunc{}
+	newDefaultDecoder  newDecoderFunc = newSchemaLessDecoder
+)
+
+func registerDecoder(schema string, dec newDecoderFunc) error {
+	if schema == "" {
+		return errors.New("schema can't be empty")
+	}
+
+	if dec == nil {
+		return errors.New("decoder can't be nil")
+	}
+
+	if _, found := registeredDecoders[schema]; found {
+		return errors.New("already registered")
+	}
+
+	registeredDecoders[schema] = dec
+
+	return nil
+}
+
+func newDecoder(cfg decodeXMLConfig) decoder {
+	newDec, found := registeredDecoders[cfg.Schema]
+	if !found {
+		return newDefaultDecoder(cfg)
+	}
+	return newDec(cfg)
+}
+
+func registerDecoders() {
+	log := logp.L().Named(logName)
+	log.Debug(registerDecoder(wineventlogSchema, newWineventlogDecoder))
+}
+
+func newSchemaLessDecoder(cfg decodeXMLConfig) decoder {
+	return func(p []byte) (common.MapStr, error) {
+		dec := xml.NewDecoder(bytes.NewReader(p))
+		if cfg.ToLower {
+			dec.LowercaseKeys()
+		}
+
+		out, err := dec.Decode()
+		if err != nil {
+			return nil, err
+		}
+
+		return common.MapStr(out), nil
+	}
+}
+
+func newWineventlogDecoder(decodeXMLConfig) decoder {
+	return func(p []byte) (common.MapStr, error) {
+		evt, err := winevent.UnmarshalXML(p)
+		if err != nil {
+			return nil, err
+		}
+		return evt.Fields(), nil
+	}
+}