diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index dfbf40767faa..46445bf640e4 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -495,8 +495,8 @@ def extract_fields(doc_list, name): if "name" not in field: continue - # Chain together names - if name != "": + # Chain together names. Names in group `base` are top-level. + if name != "" and name != "base": newName = name + "." + field["name"] else: newName = field["name"] diff --git a/x-pack/auditbeat/module/system/host/_meta/data.json b/x-pack/auditbeat/module/system/host/_meta/data.json index 341a03c3b7a9..e0b0818dcae8 100644 --- a/x-pack/auditbeat/module/system/host/_meta/data.json +++ b/x-pack/auditbeat/module/system/host/_meta/data.json @@ -10,6 +10,7 @@ "module": "system", "kind": "state" }, + "message": "Ubuntu host ubuntu-bionic (IP: 10.0.2.15) is up for 0 days, 5 hours, 11 minutes", "service": { "type": "system" }, @@ -17,9 +18,10 @@ "audit": { "host": { "architecture": "x86_64", - "boottime": "2018-12-04T12:13:02Z", + "boottime": "2018-12-10T15:48:44Z", "containerized": false, - "id": "b0d3f38d51bdeefe224737595c03d916", + "hostname": "ubuntu-bionic", + "id": "6f7be6fb33e6c77f057266415c094408", "ip": [ "10.0.2.15", "fe80::2d:fdff:fe81:e747", @@ -36,17 +38,16 @@ "02:42:83:be:1a:3a", "02:42:9e:d3:d8:88" ], - "hostname": "ubuntu-bionic", "os": { "family": "debian", - "kernel": "4.15.0-39-generic", + "kernel": "4.15.0-42-generic", "name": "Ubuntu", "platform": "ubuntu", "version": "18.04.1 LTS (Bionic Beaver)" }, "timezone.name": "UTC", "timezone.offset.sec": 0, - "uptime": 105705490232434 + "uptime": 18661357350265 } } } diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index 2d61a0c81cdd..2b7e2b2817c6 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -8,7 +8,9 @@ import ( "bytes" "encoding/binary" "encoding/gob" + "fmt" "io" + "math" "net" "strconv" "time" @@ -41,7 +43,7 @@ const ( type eventAction uint8 const ( - eventActionHost eventAction = iota + 1 + eventActionHost eventAction = iota eventActionIDChanged eventActionReboot eventActionHostnameChanged @@ -123,12 +125,7 @@ func (host *Host) toMapStr() common.MapStr { var ipStrings []string for _, addr := range host.addrs { - switch v := addr.(type) { - case *net.IPNet: - ipStrings = append(ipStrings, v.IP.String()) - case *net.IPAddr: - ipStrings = append(ipStrings, v.IP.String()) - } + ipStrings = append(ipStrings, ipString(addr)) } mapstr.Put("ip", ipStrings) @@ -144,6 +141,17 @@ func (host *Host) toMapStr() common.MapStr { return mapstr } +func ipString(addr net.Addr) string { + switch v := addr.(type) { + case *net.IPNet: + return v.IP.String() + case *net.IPAddr: + return v.IP.String() + default: + return "" + } +} + func init() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), @@ -317,11 +325,64 @@ func hostEvent(host *Host, eventType string, action eventAction) mb.Event { "kind": eventType, "action": action.String(), }, + "message": hostMessage(host, action), }, MetricSetFields: host.toMapStr(), } } +func hostMessage(host *Host, action eventAction) string { + var firstIP string + if len(host.addrs) > 0 { + firstIP = ipString(host.addrs[0]) + } + + // Hostname + IP of the first non-loopback interface. + hostString := fmt.Sprintf("%v (IP: %v)", host.info.Hostname, firstIP) + + var message string + switch action { + case eventActionHost: + message = fmt.Sprintf("%v host %v is up for %v", + host.info.OS.Name, hostString, fmtDuration(host.uptime)) + case eventActionIDChanged: + message = fmt.Sprintf("ID of host %v has changed", hostString) + case eventActionReboot: + message = fmt.Sprintf("Host %v restarted", hostString) + case eventActionHostnameChanged: + message = fmt.Sprintf("Hostname changed to %v", hostString) + case eventActionHostChanged: + message = fmt.Sprintf("Host %v changed", hostString) + } + + return message +} + +func fmtDuration(d time.Duration) string { + const dayMinutes = 60 * 24 + + remainingMinutes := math.Floor(d.Minutes()) + days := math.Floor(remainingMinutes / dayMinutes) + + remainingMinutes -= days * dayMinutes + hours := math.Floor(remainingMinutes / 60) + + remainingMinutes -= hours * 60 + minutes := math.Floor(remainingMinutes) + + return fmt.Sprintf("%.f %v, %.f %v, %.f %v", + days, inflect("day", int(days)), + hours, inflect("hour", int(hours)), + minutes, inflect("minute", int(minutes))) +} + +func inflect(noun string, count int) string { + if count == 1 { + return noun + } + return noun + "s" +} + func (ms *MetricSet) saveStateToDisk() error { var buf bytes.Buffer encoder := gob.NewEncoder(&buf) diff --git a/x-pack/auditbeat/module/system/process/_meta/data.json b/x-pack/auditbeat/module/system/process/_meta/data.json index ee8670bce941..810a310ad728 100644 --- a/x-pack/auditbeat/module/system/process/_meta/data.json +++ b/x-pack/auditbeat/module/system/process/_meta/data.json @@ -7,19 +7,23 @@ "event": { "action": "existing_process", "dataset": "process", - "id": "203e3d86-6b94-4e36-b906-930187073b93", + "id": "5795d53b-f7c2-463c-9c04-f316ae876d51", "module": "system", "kind": "state" }, + "message": "Process zsh (PID: 2363) is RUNNING", "process": { "args": [ - "/sbin/init" + "/usr/bin/zsh" ], - "executable": "/lib/systemd/systemd", - "name": "systemd", - "pid": 1, - "ppid": 0, - "start": "2018-12-03T23:49:23.08Z", - "working_directory": "/" + "executable": "/bin/zsh", + "name": "zsh", + "pid": 2363, + "ppid": 2362, + "start": "2018-12-10T16:36:25.21Z", + "working_directory": "/home/elastic" + }, + "service": { + "type": "system" } } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index 55197098c37a..b8754d45d493 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -5,6 +5,7 @@ package process import ( + "fmt" "strconv" "time" @@ -33,12 +34,29 @@ const ( eventTypeState = "state" eventTypeEvent = "event" +) + +type eventAction uint8 - eventActionExistingProcess = "existing_process" - eventActionProcessStarted = "process_started" - eventActionProcessStopped = "process_stopped" +const ( + eventActionExistingProcess eventAction = iota + eventActionProcessStarted + eventActionProcessStopped ) +func (action eventAction) String() string { + switch action { + case eventActionExistingProcess: + return "existing_process" + case eventActionProcessStarted: + return "process_started" + case eventActionProcessStopped: + return "process_stopped" + default: + return "" + } +} + func init() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), @@ -214,18 +232,34 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { return nil } -func processEvent(pInfo *ProcessInfo, eventType string, eventAction string) mb.Event { +func processEvent(pInfo *ProcessInfo, eventType string, action eventAction) mb.Event { return mb.Event{ RootFields: common.MapStr{ "event": common.MapStr{ "kind": eventType, - "action": eventAction, + "action": action.String(), }, "process": pInfo.toMapStr(), + "message": processMessage(pInfo, action), }, } } +func processMessage(pInfo *ProcessInfo, action eventAction) string { + var actionString string + switch action { + case eventActionProcessStarted: + actionString = "STARTED" + case eventActionProcessStopped: + actionString = "STOPPED" + case eventActionExistingProcess: + actionString = "is RUNNING" + } + + return fmt.Sprintf("Process %v (PID: %d) %v", + pInfo.Name, pInfo.PID, actionString) +} + func convertToCacheable(processInfos []*ProcessInfo) []cache.Cacheable { c := make([]cache.Cacheable, 0, len(processInfos)) diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index 91876cd0e2d2..6de5dd97ea44 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -24,7 +24,10 @@ func TestData(t *testing.T) { t.Fatal("no events were generated") } - fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent) + // The first process (events[0]) is usually something like systemd, + // the last few are test processes, so we pick something more interesting + // towards the end. + fullEvent := mbtest.StandardizeEvent(f, events[len(events)-8], core.AddDatasetToEvent) mbtest.WriteEventToDataJSON(t, fullEvent, "") } diff --git a/x-pack/auditbeat/module/system/socket/_meta/data.json b/x-pack/auditbeat/module/system/socket/_meta/data.json index fea6c02bb564..a373c3cf1796 100644 --- a/x-pack/auditbeat/module/system/socket/_meta/data.json +++ b/x-pack/auditbeat/module/system/socket/_meta/data.json @@ -4,30 +4,35 @@ "hostname": "host.example.com", "name": "host.example.com" }, - "user": { - "name": "vagrant", - "id": 1000 - }, - "process": { - "pid": 4021, - "name": "lynx" - }, - "source": { - "ip": "10.0.2.15", - "port": 33956 - }, "destination": { - "port": 443, - "ip": "52.10.168.186" + "ip": "10.0.2.15", + "port": 22 }, "event": { - "kind": "event", - "action": "socket_opened", - "module": "system", - "dataset": "socket" + "action": "existing_socket", + "dataset": "socket", + "id": "6aff69f8-7267-4604-9701-d7b67a7c65bc", + "kind": "state", + "module": "system" }, + "message": "Inbound socket (10.0.2.2:55270 -\u003e 10.0.2.15:22) OPEN by process sshd (PID: 22799) and user root (UID: 0)", "network": { - "type": "ipv4", - "direction": "outbound" + "direction": "inbound", + "type": "ipv4" + }, + "process": { + "name": "sshd", + "pid": 22799 + }, + "service": { + "type": "system" + }, + "source": { + "ip": "10.0.2.2", + "port": 55270 + }, + "user": { + "id": 0, + "name": "root" } -} +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/socket/socket.go b/x-pack/auditbeat/module/system/socket/socket.go index de8db36d76a1..b69148e45290 100644 --- a/x-pack/auditbeat/module/system/socket/socket.go +++ b/x-pack/auditbeat/module/system/socket/socket.go @@ -11,6 +11,7 @@ import ( "net" "os/user" "strconv" + "strings" "syscall" "time" @@ -34,12 +35,29 @@ const ( eventTypeState = "state" eventTypeEvent = "event" +) + +type eventAction uint8 - eventActionExistingSocket = "existing_socket" - eventActionSocketOpened = "socket_opened" - eventActionSocketClosed = "socket_closed" +const ( + eventActionExistingSocket eventAction = iota + eventActionSocketOpened + eventActionSocketClosed ) +func (action eventAction) String() string { + switch action { + case eventActionExistingSocket: + return "existing_socket" + case eventActionSocketOpened: + return "socket_opened" + case eventActionSocketClosed: + return "socket_closed" + default: + return "" + } +} + func init() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), @@ -106,7 +124,8 @@ func (s Socket) Hash() uint64 { func (s Socket) toMapStr() common.MapStr { mapstr := common.MapStr{ "network": common.MapStr{ - "type": s.Family.String(), + "type": s.Family.String(), + "direction": ecsDirectionString(s.Direction), }, "user": common.MapStr{ "id": s.UID, @@ -126,7 +145,6 @@ func (s Socket) toMapStr() common.MapStr { switch s.Direction { case sock.Outgoing: - mapstr.Put("network.direction", "outbound") mapstr.Put("source", common.MapStr{ "ip": s.LocalIP, "port": s.LocalPort, @@ -136,7 +154,6 @@ func (s Socket) toMapStr() common.MapStr { "port": s.RemotePort, }) case sock.Incoming: - mapstr.Put("network.direction", "inbound") mapstr.Put("source", common.MapStr{ "ip": s.RemoteIP, "port": s.RemotePort, @@ -146,7 +163,6 @@ func (s Socket) toMapStr() common.MapStr { "port": s.LocalPort, }) case sock.Listening: - mapstr.Put("network.direction", "listening") mapstr.Put("destination", common.MapStr{ "ip": s.LocalIP, "port": s.LocalPort, @@ -160,6 +176,21 @@ func (s Socket) toMapStr() common.MapStr { return mapstr } +// ecsDirectionString is a custom alternative to the existing String() +// to be compatible with recommended ECS values for network.direction. +func ecsDirectionString(direction sock.Direction) string { + switch direction { + case sock.Incoming: + return "inbound" + case sock.Outgoing: + return "outbound" + case sock.Listening: + return "listening" + default: + return "unknown" + } +} + // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) @@ -274,17 +305,46 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { return nil } -func socketEvent(socket *Socket, eventType string, eventAction string) mb.Event { +func socketEvent(socket *Socket, eventType string, action eventAction) mb.Event { event := mb.Event{ RootFields: socket.toMapStr(), } event.RootFields.Put("event.kind", eventType) - event.RootFields.Put("event.action", eventAction) + event.RootFields.Put("event.action", action.String()) + event.RootFields.Put("message", socketMessage(socket, action)) return event } +func socketMessage(socket *Socket, action eventAction) string { + var actionString string + switch action { + case eventActionSocketOpened: + actionString = "OPENED" + case eventActionSocketClosed: + actionString = "CLOSED" + case eventActionExistingSocket: + actionString = "OPEN" + } + + var endpointString string + switch socket.Direction { + case sock.Incoming: + endpointString = fmt.Sprintf("%v:%d -> %v:%d", + socket.RemoteIP, socket.RemotePort, socket.LocalIP, socket.LocalPort) + case sock.Outgoing: + endpointString = fmt.Sprintf("%v:%d -> %v:%d", + socket.LocalIP, socket.LocalPort, socket.RemoteIP, socket.RemotePort) + case sock.Listening: + endpointString = fmt.Sprintf("%v:%d", socket.LocalIP, socket.LocalPort) + } + + return fmt.Sprintf("%v socket (%v) %v by process %v (PID: %d) and user %v (UID: %d)", + strings.Title(ecsDirectionString(socket.Direction)), endpointString, actionString, + socket.ProcessName, socket.ProcessPID, socket.Username, socket.UID) +} + func convertToCacheable(sockets []*Socket) []cache.Cacheable { c := make([]cache.Cacheable, 0, len(sockets)) diff --git a/x-pack/auditbeat/module/system/socket/socket_test.go b/x-pack/auditbeat/module/system/socket/socket_test.go index e7824cab83b0..cc88fadbd921 100644 --- a/x-pack/auditbeat/module/system/socket/socket_test.go +++ b/x-pack/auditbeat/module/system/socket/socket_test.go @@ -29,7 +29,10 @@ func TestData(t *testing.T) { if len(events) == 0 { t.Fatal("no events were generated") } - fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent) + + // The first socket (events[0]) is usually something like rpcbind, + // the last one should be more interesting. + fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent) mbtest.WriteEventToDataJSON(t, fullEvent, "") } diff --git a/x-pack/auditbeat/module/system/user/_meta/data.json b/x-pack/auditbeat/module/system/user/_meta/data.json index f79b9ea904c1..5958f7288369 100644 --- a/x-pack/auditbeat/module/system/user/_meta/data.json +++ b/x-pack/auditbeat/module/system/user/_meta/data.json @@ -1,40 +1,48 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "event": { - "module": "system", + "action": "existing_user", "dataset": "user", + "id": "b0bbc4e2-9540-4aaf-b74e-ef9de506e852", "kind": "state", - "action": "existing_user", - "id": "57ee8bb6-a3da-4c43-b0d9-0688ccdc88d0" + "module": "system" }, - "user": { - "id": 1001, - "name": "ubuntu" + "message": "Existing user elastic (UID: 1002, Groups: elastic,docker)", + "service": { + "type": "system" }, "system": { "audit": { "user": { - "uid": 1001, - "gid": 1001, - "name": "ubuntu", - "dir": "/home/ubuntu", - "shell": "/bin/bash", - "user_information": "Ubuntu", + "dir": "/home/elastic", + "gid": 1002, "group": [ { - "name": "sudo", - "gid": 27 + "gid": 1002, + "name": "elastic" + }, + { + "gid": 999, + "name": "docker" } ], + "name": "elastic", "password": { - "type": "shadow_password", - "last_changed": "2018-09-21T00:00:00.000Z" - } + "last_changed": "2018-12-07T00:00:00Z", + "type": "shadow_password" + }, + "shell": "/usr/bin/zsh", + "uid": 1002, + "user_information": ",,," } } + }, + "user": { + "id": 1002, + "name": "elastic" } -} +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/user/user.go b/x-pack/auditbeat/module/system/user/user.go index 6b771cb88282..5cc12d5f4b07 100644 --- a/x-pack/auditbeat/module/system/user/user.go +++ b/x-pack/auditbeat/module/system/user/user.go @@ -14,6 +14,7 @@ import ( "io" "runtime" "strconv" + "strings" "syscall" "time" @@ -44,14 +45,35 @@ const ( eventTypeState = "state" eventTypeEvent = "event" +) + +type eventAction uint8 - eventActionExistingUser = "existing_user" - eventActionUserAdded = "user_added" - eventActionUserRemoved = "user_removed" - eventActionUserChanged = "user_changed" - eventActionPasswordChanged = "password_changed" +const ( + eventActionExistingUser eventAction = iota + eventActionUserAdded + eventActionUserRemoved + eventActionUserChanged + eventActionPasswordChanged ) +func (action eventAction) String() string { + switch action { + case eventActionExistingUser: + return "existing_user" + case eventActionUserAdded: + return "user_added" + case eventActionUserRemoved: + return "user_removed" + case eventActionUserChanged: + return "user_changed" + case eventActionPasswordChanged: + return "password_changed" + default: + return "" + } +} + type passwordType uint8 const ( @@ -383,22 +405,54 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { return nil } -func userEvent(user *User, eventType string, eventAction string) mb.Event { +func userEvent(user *User, eventType string, action eventAction) mb.Event { return mb.Event{ RootFields: common.MapStr{ "event": common.MapStr{ "kind": eventType, - "action": eventAction, + "action": action.String(), }, "user": common.MapStr{ "id": user.UID, "name": user.Name, }, + "message": userMessage(user, action), }, MetricSetFields: user.toMapStr(), } } +func userMessage(user *User, action eventAction) string { + var actionString string + switch action { + case eventActionExistingUser: + actionString = "Existing" + case eventActionUserAdded: + actionString = "New" + case eventActionUserRemoved: + actionString = "Removed" + case eventActionUserChanged: + actionString = "Changed" + case eventActionPasswordChanged: + actionString = "Password changed for" + } + + return fmt.Sprintf("%v user %v (UID: %v, Groups: %v)", + actionString, user.Name, user.UID, fmtGroups(user.Groups)) +} + +func fmtGroups(groups []Group) string { + var b strings.Builder + + b.WriteString(groups[0].Name) + for _, group := range groups[1:] { + b.WriteString(",") + b.WriteString(group.Name) + } + + return b.String() +} + func convertToCacheable(users []*User) []cache.Cacheable { c := make([]cache.Cacheable, 0, len(users)) diff --git a/x-pack/auditbeat/module/system/user/user_test.go b/x-pack/auditbeat/module/system/user/user_test.go index ccb606f4d24d..f02f6bc3d8f2 100644 --- a/x-pack/auditbeat/module/system/user/user_test.go +++ b/x-pack/auditbeat/module/system/user/user_test.go @@ -24,13 +24,15 @@ func TestData(t *testing.T) { t.Fatal("no events were generated") } - fullEvent := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent) + // The first user (events[0]) is usually root, the last one should be more interesting. + fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent) mbtest.WriteEventToDataJSON(t, fullEvent, "") } func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", - "metricsets": []string{"user"}, + "module": "system", + "metricsets": []string{"user"}, + "user.detect_password_changes": true, } }