[libbeat] Decode xml enhance wineventlog

libbeat/processors/decode_xml/schema.go

@@ -24,13 +24,15 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/encoding/xml"
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/winlogbeat/sys/winevent"
 )
 
 const wineventlogSchema = "wineventlog"
 
 type newDecoderFunc func(cfg decodeXMLConfig) decoder
-type decoder func(p []byte) (common.MapStr, error)
+
+// a decoder will decode the raw XML into a set of fields,
+// optionally will return additional ECS mappings when possible
+type decoder func(p []byte) (decodedXML common.MapStr, err error)
 
 var (
 	registeredDecoders                = map[string]newDecoderFunc{}
@@ -83,13 +85,3 @@ func newSchemaLessDecoder(cfg decodeXMLConfig) decoder {
 		return common.MapStr(out), nil
 	}
 }
-
-func newWineventlogDecoder(decodeXMLConfig) decoder {
-	return func(p []byte) (common.MapStr, error) {
-		evt, err := winevent.UnmarshalXML(p)
-		if err != nil {
-			return nil, err
-		}
-		return evt.Fields(), nil
-	}
-}

libbeat/processors/decode_xml/schema_test.go

@@ -40,9 +40,10 @@ func TestDecodeSchemas(t *testing.T) {
 		{
 			schema: wineventlogSchema,
 			config: decodeXMLConfig{
-				Field:  "message",
-				Target: &testXMLTargetField,
-				Schema: wineventlogSchema,
+				Field:         "message",
+				Target:        &testXMLTargetField,
+				Schema:        wineventlogSchema,
+				OverwriteKeys: true,
 			},
 			Input: common.MapStr{
 				"message": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Security-Auditing' Guid='{54849625-5478-4994-a5ba-3e3b0328c30d}'/>" +

libbeat/processors/decode_xml/schema_wineventlog.go

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build !windows
+
+package decode_xml
+
+import (
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/winlogbeat/sys/winevent"
+)
+
+func newWineventlogDecoder(decodeXMLConfig) decoder {
+	return func(p []byte) (common.MapStr, error) {
+		evt, err := winevent.UnmarshalXML(p)
+		if err != nil {
+			return nil, err
+		}
+		winevent.EnrichRawValuesWithNames(nil, &evt)
+		return evt.Fields(), nil
+	}
+}

libbeat/processors/decode_xml/schema_wineventlog_windows.go

@@ -0,0 +1,84 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build windows
+
+package decode_xml
+
+import (
+	"sync"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/winlogbeat/sys/winevent"
+	"github.com/elastic/beats/v7/winlogbeat/sys/wineventlog"
+)
+
+func newWineventlogDecoder(decodeXMLConfig) decoder {
+	cache := &metadataCache{
+		store: map[string]*winevent.WinMeta{},
+	}
+	return func(p []byte) (common.MapStr, error) {
+		evt, err := winevent.UnmarshalXML(p)
+		if err != nil {
+			return nil, err
+		}
+		md := cache.getPublisherMetadata(evt.Provider.Name)
+		winevent.EnrichRawValuesWithNames(md, &evt)
+		return evt.Fields(), nil
+	}
+}
+
+type metadataCache struct {
+	store map[string]*winevent.WinMeta
+	mutex sync.RWMutex
+}
+
+func (c *metadataCache) getPublisherMetadata(publisher string) *winevent.WinMeta {
+	// NOTE: This code uses double-check locking to elevate to a write-lock
+	// when a cache value needs initialized.
+	c.mutex.RLock()
+
+	log := logp.L().Named(logName)
+	// Lookup cached value.
+	md, found := c.store[publisher]
+	if !found {
+		// Elevate to write lock.
+		c.mutex.RUnlock()
+		c.mutex.Lock()
+		defer c.mutex.Unlock()
+
+		// Double-check if the condition changed while upgrading the lock.
+		md, found = c.store[publisher]
+		if found {
+			return md
+		}
+
+		// Load metadata from the publisher.
+		md, err := wineventlog.NewPublisherMetadataStore(wineventlog.NilHandle, publisher, log)
+		if err != nil {
+			// Return an empty store on error (can happen in cases where the
+			// log was forwarded and the provider doesn't exist on collector).
+			md = wineventlog.NewEmptyPublisherMetadataStore(publisher, log)
+		}
+		c.store[publisher] = &md.WinMeta
+	} else {
+		c.mutex.RUnlock()
+	}
+
+	return md
+}

winlogbeat/sys/winevent/event.go

@@ -44,6 +44,11 @@ var (
 const (
 	keywordAuditFailure = 0x10000000000000
 	keywordAuditSuccess = 0x20000000000000
+
+	// keywordClassic indicates the log was published with the "classic" event
+	// logging API.
+	// https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.eventing.reader.standardeventkeywords?view=netframework-4.8
+	keywordClassic = 0x80000000000000
 )
 
 // UnmarshalXML unmarshals the given XML into a new Event.
@@ -101,7 +106,6 @@ func (e Event) Fields() common.MapStr {
 	AddOptional(win, "keywords", e.Keywords)
 	AddOptional(win, "opcode", e.Opcode)
 	AddOptional(win, "provider_guid", e.Provider.GUID)
-	AddOptional(win, "task", e.Task)
 	AddOptional(win, "version", e.Version)
 	AddOptional(win, "time_created", e.TimeCreated.SystemTime)
 
@@ -330,3 +334,60 @@ func (v *HexInt64) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	*v = HexInt64(num)
 	return nil
 }
+
+// EnrichRawValuesWithNames adds the names associated with the raw system
+// property values. It enriches the event with keywords, opcode, level, and
+// task. The search order is defined in the EvtFormatMessage documentation.
+func EnrichRawValuesWithNames(publisherMeta *WinMeta, event *Event) {
+	// Keywords. Each bit in the value can represent a keyword.
+	rawKeyword := int64(event.KeywordsRaw)
+	isClassic := keywordClassic&rawKeyword > 0
+
+	if len(event.Keywords) == 0 {
+		for mask, keyword := range defaultWinMeta.Keywords {
+			if rawKeyword&mask > 0 {
+				event.Keywords = append(event.Keywords, keyword)
+				rawKeyword -= mask
+			}
+		}
+		if publisherMeta != nil {
+			for mask, keyword := range publisherMeta.Keywords {
+				if rawKeyword&mask > 0 {
+					event.Keywords = append(event.Keywords, keyword)
+					rawKeyword -= mask
+				}
+			}
+		}
+	}
+
+	var found bool
+	if event.Opcode == "" {
+		// Opcode (search in defaultWinMeta first).
+		if !isClassic {
+			event.Opcode, found = defaultWinMeta.Opcodes[event.OpcodeRaw]
+			if !found && publisherMeta != nil {
+				event.Opcode = publisherMeta.Opcodes[event.OpcodeRaw]
+			}
+		}
+	}
+
+	if event.Level == "" {
+		// Level (search in defaultWinMeta first).
+		event.Level, found = defaultWinMeta.Levels[event.LevelRaw]
+		if !found && publisherMeta != nil {
+			event.Level = publisherMeta.Levels[event.LevelRaw]
+		}
+	}
+
+	if event.Task == "" {
+		if publisherMeta != nil {
+			// Task (fall-back to defaultWinMeta if not found).
+			event.Task, found = publisherMeta.Tasks[event.TaskRaw]
+			if !found {
+				event.Task = defaultWinMeta.Tasks[event.TaskRaw]
+			}
+		} else {
+			event.Task = defaultWinMeta.Tasks[event.TaskRaw]
+		}
+	}
+}

winlogbeat/sys/winevent/maputil.go

@@ -80,6 +80,10 @@ func AddPairs(m common.MapStr, key string, pairs []KeyValue) common.MapStr {
 
 // isZero return true if the given value is the zero value for its type.
 func isZero(i interface{}) bool {
+	if i == nil {
+		return true
+	}
+
 	v := reflect.ValueOf(i)
 	switch v.Kind() {
 	case reflect.Array, reflect.String:
@@ -95,5 +99,6 @@ func isZero(i interface{}) bool {
 	case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
 		return v.IsNil()
 	}
+
 	return false
 }

winlogbeat/sys/wineventlog/winmeta.go → winlogbeat/sys/winevent/winmeta.go

@@ -15,13 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// +build windows
+package winevent
 
-package wineventlog
+type WinMeta struct {
+	Keywords map[int64]string  // Keyword bit mask to keyword name.
+	Opcodes  map[uint8]string  // Opcode value to name.
+	Levels   map[uint8]string  // Level value to name.
+	Tasks    map[uint16]string // Task value to name.
+}
 
-// winMeta contains the static values that are a common across Windows. These
+// defaultWinMeta contains the static values that are a common across Windows. These
 // values are from winmeta.xml inside the Windows SDK.
-var winMeta = &publisherMetadataStore{
+var defaultWinMeta = &WinMeta{
 	Keywords: map[int64]string{
 		0:                "AnyKeyword",
 		0x1000000000000:  "Response Time",