diff --git a/libbeat/cmd/instance/imports_common.go b/libbeat/cmd/instance/imports_common.go index ac767a559645..724919364db1 100644 --- a/libbeat/cmd/instance/imports_common.go +++ b/libbeat/cmd/instance/imports_common.go @@ -31,6 +31,7 @@ import ( _ "github.com/elastic/beats/v7/libbeat/processors/communityid" _ "github.com/elastic/beats/v7/libbeat/processors/convert" _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml" + _ "github.com/elastic/beats/v7/libbeat/processors/decode_xml_wineventlog" _ "github.com/elastic/beats/v7/libbeat/processors/dissect" _ "github.com/elastic/beats/v7/libbeat/processors/dns" _ "github.com/elastic/beats/v7/libbeat/processors/extract_array" diff --git a/libbeat/processors/decode_xml/config.go b/libbeat/processors/decode_xml/config.go index 289b2eaa0e97..21bb426c5b2f 100644 --- a/libbeat/processors/decode_xml/config.go +++ b/libbeat/processors/decode_xml/config.go @@ -25,6 +25,7 @@ type decodeXMLConfig struct { ToLower bool `config:"to_lower"` IgnoreMissing bool `config:"ignore_missing"` IgnoreFailure bool `config:"ignore_failure"` + Schema string `config:"schema"` } func defaultConfig() decodeXMLConfig { diff --git a/libbeat/processors/decode_xml/decode_xml.go b/libbeat/processors/decode_xml/decode_xml.go index 0b229cff3d23..99f14ad1d182 100644 --- a/libbeat/processors/decode_xml/decode_xml.go +++ b/libbeat/processors/decode_xml/decode_xml.go @@ -21,12 +21,10 @@ import ( "encoding/json" "errors" "fmt" - "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/common/encoding/xml" "github.com/elastic/beats/v7/libbeat/common/jsontransform" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -36,7 +34,9 @@ import ( type decodeXML struct { decodeXMLConfig - log *logp.Logger + + decode decoder + log *logp.Logger } var ( @@ -51,9 +51,15 @@ const ( func init() { processors.RegisterPlugin(procName, checks.ConfigChecked(New, - checks.RequireFields("fields"), - checks.AllowedFields("fields", "overwrite_keys", "add_error_key", "target", "document_id"))) + checks.RequireFields("field"), + checks.AllowedFields( + "field", "target_field", + "overwrite_keys", "document_id", + "to_lower", "ignore_missing", + "ignore_failure", "schema", + ))) jsprocessor.RegisterPlugin(procName, New) + registerDecoders() } // New constructs a new decode_xml processor. @@ -77,6 +83,7 @@ func newDecodeXML(config decodeXMLConfig) (processors.Processor, error) { return &decodeXML{ decodeXMLConfig: config, + decode: newDecoder(config), log: logp.NewLogger(logName), }, nil } @@ -104,7 +111,7 @@ func (x *decodeXML) run(event *beat.Event) error { return errFieldIsNotString } - xmlOutput, err := x.decodeField(text) + xmlOutput, err := x.decode([]byte(text)) if err != nil { return err } @@ -131,19 +138,6 @@ func (x *decodeXML) run(event *beat.Event) error { return nil } -func (x *decodeXML) decodeField(data string) (decodedData map[string]interface{}, err error) { - dec := xml.NewDecoder(strings.NewReader(data)) - if x.ToLower { - dec.LowercaseKeys() - } - - out, err := dec.Decode() - if err != nil { - return nil, fmt.Errorf("error decoding XML field: %w", err) - } - return out, nil -} - func (x *decodeXML) String() string { json, _ := json.Marshal(x.decodeXMLConfig) return procName + "=" + string(json) diff --git a/libbeat/processors/decode_xml/decode_xml_test.go b/libbeat/processors/decode_xml/decode_xml_test.go index 26d075bf3a47..7b8b2758c0e7 100644 --- a/libbeat/processors/decode_xml/decode_xml_test.go +++ b/libbeat/processors/decode_xml/decode_xml_test.go @@ -57,7 +57,7 @@ func TestDecodeXML(t *testing.T) { `, }, Output: common.MapStr{ - "xml": map[string]interface{}{ + "xml": common.MapStr{ "catalog": map[string]interface{}{ "book": map[string]interface{}{ "author": "William H. Gaddis", @@ -125,7 +125,7 @@ func TestDecodeXML(t *testing.T) { `, }, Output: common.MapStr{ - "message": map[string]interface{}{ + "message": common.MapStr{ "catalog": map[string]interface{}{ "book": map[string]interface{}{ "author": "William H. Gaddis", @@ -158,7 +158,7 @@ func TestDecodeXML(t *testing.T) { `, }, Output: common.MapStr{ - "message": map[string]interface{}{ + "message": common.MapStr{ "catalog": map[string]interface{}{ "book": []interface{}{ map[string]interface{}{ @@ -203,7 +203,7 @@ func TestDecodeXML(t *testing.T) { `, }, Output: common.MapStr{ - "message": map[string]interface{}{ + "message": common.MapStr{ "catalog": map[string]interface{}{ "book": []interface{}{ map[string]interface{}{ diff --git a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc index ded0543514ac..3ec903730f3b 100644 --- a/libbeat/processors/decode_xml/docs/decode_xml.asciidoc +++ b/libbeat/processors/decode_xml/docs/decode_xml.asciidoc @@ -55,15 +55,13 @@ Example XML input: [source,xml] ------------------------------------------------------------------------------- -{ - - - William H. Gaddis - The Recognitions - One of the great seminal American novels of the 20th century. - - -} + + + William H. Gaddis + The Recognitions + One of the great seminal American novels of the 20th century. + + ------------------------------------------------------------------------------- Will produce the following output: @@ -97,10 +95,13 @@ value (`target_field:`) is treated as if the field was not set at all. `overwrite_keys`:: (Optional) A boolean that specifies whether keys that already exist in the event are overwritten by keys from the decoded XML object. The -default value is false. +default value is `true`. `to_lower`:: (Optional) Converts all keys to lowercase. Accepts either true or -false. The default value is true. +false. The default value is `true`. + +`schema`:: (Optional) Specifies the schema of the message. Accepted schemas: `wineventlog`. +The default value is ``. `document_id`:: (Optional) XML key to use as the document ID. If configured, the field will be removed from the original XML document and stored in diff --git a/libbeat/processors/decode_xml/schema.go b/libbeat/processors/decode_xml/schema.go new file mode 100644 index 000000000000..9f1627041725 --- /dev/null +++ b/libbeat/processors/decode_xml/schema.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml + +import ( + "bytes" + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/encoding/xml" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +type newDecoderFunc func(cfg decodeXMLConfig) decoder +type decoder func(p []byte) (common.MapStr, error) + +var ( + registeredDecoders = map[string]newDecoderFunc{} + newDefaultDecoder newDecoderFunc = newSchemaLessDecoder +) + +func registerDecoder(schema string, dec newDecoderFunc) error { + if schema == "" { + return errors.New("schema can't be empty") + } + + if dec == nil { + return errors.New("decoder can't be nil") + } + + if _, found := registeredDecoders[schema]; found { + return errors.New("already registered") + } + + registeredDecoders[schema] = dec + + return nil +} + +func newDecoder(cfg decodeXMLConfig) decoder { + newDec, found := registeredDecoders[cfg.Schema] + if !found { + return newDefaultDecoder(cfg) + } + return newDec(cfg) +} + +func registerDecoders() { + log := logp.L().Named(logName) + log.Debug(registerDecoder("wineventlog", newWineventlogDecoder)) +} + +func newSchemaLessDecoder(cfg decodeXMLConfig) decoder { + return func(p []byte) (common.MapStr, error) { + dec := xml.NewDecoder(bytes.NewReader(p)) + if cfg.ToLower { + dec.LowercaseKeys() + } + + out, err := dec.Decode() + if err != nil { + return nil, fmt.Errorf("error decoding XML field: %w", err) + } + + return common.MapStr(out), nil + } +} + +func newWineventlogDecoder(decodeXMLConfig) decoder { + return func(p []byte) (common.MapStr, error) { + evt, err := winevent.UnmarshalXML(p) + if err != nil { + return nil, err + } + return evt.Fields(), nil + } +} diff --git a/libbeat/processors/decode_xml_wineventlog/config.go b/libbeat/processors/decode_xml_wineventlog/config.go new file mode 100644 index 000000000000..be66da0f1cea --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/config.go @@ -0,0 +1,33 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +type config struct { + Field string `config:"field" validate:"required"` + Target *string `config:"target_field"` + OverwriteKeys bool `config:"overwrite_keys"` + IgnoreMissing bool `config:"ignore_missing"` + IgnoreFailure bool `config:"ignore_failure"` +} + +func defaultConfig() config { + return config{ + Field: "message", + OverwriteKeys: true, + } +} diff --git a/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc new file mode 100644 index 000000000000..2c1e7f40a185 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/docs/decode_xml_wineventlog.asciidoc @@ -0,0 +1,137 @@ +[[decode_xml]] +=== Decode XML + +++++ +decode_xml +++++ + +experimental[] + +The `decode_xml_wineventlog` processor decodes Windows Event data exported in XML under the `field` +key. It outputs the result into the `target_field`. + +This example demonstrates how to decode an XML string contained in the `message` +field and write the resulting fields into the root of the document. Any fields +that already exist will be overwritten. + +[source,yaml] +------- +processors: + - decode_xml_wineventlog: + field: message + target_field: "" + overwrite_keys: true +------- + +By default any decoding errors that occur will stop the processing chain and the +error will be added to `error.message` field. To ignore all errors and continue +to the next processor you can set `ignore_failure: true`. To specifically +ignore failures caused by `field` not existing use `ignore_missing`. + +[source,yaml] +------- +processors: + - decode_xml_wineventlog: + field: event.original + target_field: winlog + ignore_missing: true + ignore_failure: true +------- + +Example XML input: + +[source,xml] +------------------------------------------------------------------------------- +4672001254800x802000000000000011303SecurityvagrantS-1-5-18SYSTEMNT AUTHORITY0x3e7SeAssignPrimaryTokenPrivilege + SeTcbPrivilege + SeSecurityPrivilege + SeTakeOwnershipPrivilege + SeLoadDriverPrivilege + SeBackupPrivilege + SeRestorePrivilege + SeDebugPrivilege + SeAuditPrivilege + SeSystemEnvironmentPrivilege + SeImpersonatePrivilege + SeDelegateSessionUserImpersonatePrivilegeSpecial privileges assigned to new logon. + +Subject: + Security ID: S-1-5-18 + Account Name: SYSTEM + Account Domain: NT AUTHORITY + Logon ID: 0x3E7 + +Privileges: SeAssignPrimaryTokenPrivilege + SeTcbPrivilege + SeSecurityPrivilege + SeTakeOwnershipPrivilege + SeLoadDriverPrivilege + SeBackupPrivilege + SeRestorePrivilege + SeDebugPrivilege + SeAuditPrivilege + SeSystemEnvironmentPrivilege + SeImpersonatePrivilege + SeDelegateSessionUserImpersonatePrivilegeInformationSpecial LogonInfoSecurityMicrosoft Windows security auditing.Audit Success +------------------------------------------------------------------------------- + +Will produce the following output: + +[source,json] +------------------------------------------------------------------------------- +{ + "winlog": { + "channel": "Security", + "outcome": "success", + "activity_id": "{ffb23523-1f32-0000-c335-b2ff321fd701}", + "level": "information", + "event_id": 4672, + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 11303, + "computer_name": "vagrant", + "keywords_raw": 9232379236109516800, + "opcode": "Info", + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "event_data": { + "SubjectUserSid": "S-1-5-18", + "SubjectUserName": "SYSTEM", + "SubjectDomainName": "NT AUTHORITY", + "SubjectLogonId": "0x3e7", + "PrivilegeList": "SeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege" + }, + "task": "Special Logon", + "keywords": [ + "Audit Success" + ], + "message": "Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tSYSTEM\n\tAccount Domain:\t\tNT AUTHORITY\n\tLogon ID:\t\t0x3E7\n\nPrivileges:\t\tSeAssignPrimaryTokenPrivilege\n\t\t\tSeTcbPrivilege\n\t\t\tSeSecurityPrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeAuditPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeImpersonatePrivilege\n\t\t\tSeDelegateSessionUserImpersonatePrivilege", + "process": { + "pid": 652, + "thread": { + "id": 4660 + } + } + } +} +------------------------------------------------------------------------------- + +The supported configuration options are: + +`field`:: (Required) Source field containing the XML. Defaults to `message`. + +`target_field`:: (Optional) The field under which the decoded XML will be +written. **By default the decoded XML object replaces the field from which it was +read.** To merge the decoded XML fields into the root of the event specify +`target_field` with an empty string (`target_field: ""`). Note that the `null` +value (`target_field:`) is treated as if the field was not set at all. + +`overwrite_keys`:: (Optional) A boolean that specifies whether keys that already +exist in the event are overwritten by keys from the decoded XML object. The +default value is `true`. + +`ignore_missing`:: (Optional) If `true` the processor will not return an error +when a specified field does not exist. Defaults to `false`. + +`ignore_failure`:: (Optional) Ignore all errors produced by the processor. +Defaults to `false`. + +See <> for a list of supported conditions. diff --git a/libbeat/processors/decode_xml_wineventlog/processor.go b/libbeat/processors/decode_xml_wineventlog/processor.go new file mode 100644 index 000000000000..29568d591da7 --- /dev/null +++ b/libbeat/processors/decode_xml_wineventlog/processor.go @@ -0,0 +1,134 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package decode_xml_wineventlog + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/jsontransform" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" + "github.com/elastic/beats/v7/winlogbeat/sys/winevent" +) + +type processor struct { + config + log *logp.Logger +} + +var ( + errFieldIsNotString = errors.New("field value is not a string") +) + +const ( + procName = "decode_xml_wineventlog" + logName = "processor." + procName +) + +func init() { + processors.RegisterPlugin(procName, + checks.ConfigChecked(New, + checks.RequireFields("field"), + checks.AllowedFields( + "field", "overwrite_keys", + "target_field", "ignore_missing", + "ignore_failure", + ))) + jsprocessor.RegisterPlugin(procName, New) +} + +// New constructs a new decode_xml processor. +func New(c *common.Config) (processors.Processor, error) { + config := defaultConfig() + + if err := c.Unpack(&config); err != nil { + return nil, fmt.Errorf("fail to unpack the "+procName+" processor configuration: %s", err) + } + + return newProcessor(config) +} + +func newProcessor(config config) (processors.Processor, error) { + // Default target to overwriting field. + if config.Target == nil { + config.Target = &config.Field + } + + return &processor{ + config: config, + log: logp.NewLogger(logName), + }, nil +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + if err := p.run(event); err != nil && !p.IgnoreFailure { + err = fmt.Errorf("failed in decode_xml_wineventlog on the %q field: %w", p.Field, err) + _, _ = event.PutValue("error.message", err.Error()) + return event, err + } + return event, nil +} + +func (p *processor) run(event *beat.Event) error { + data, err := event.GetValue(p.Field) + if err != nil { + if p.IgnoreMissing && err == common.ErrKeyNotFound { + return nil + } + return err + } + + text, ok := data.(string) + if !ok { + return errFieldIsNotString + } + + winevt, err := p.decode(text) + if err != nil { + return err + } + + if *p.Target != "" { + if _, err = event.PutValue(*p.Target, winevt); err != nil { + return fmt.Errorf("failed to put value %v into field %q: %w", winevt, *p.Target, err) + } + } else { + jsontransform.WriteJSONKeys(event, winevt, false, p.OverwriteKeys, !p.IgnoreFailure) + } + + return nil +} + +func (p *processor) decode(data string) (common.MapStr, error) { + evt, err := winevent.UnmarshalXML([]byte(data)) + if err != nil { + return nil, err + } + return evt.Fields(), nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +}