[microsoft_dhcp] Format observer.mac per ECS

packages/microsoft_dhcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.1"
+  changes:
+    - description: Format observer.mac as per ECS and add missing mappings for event.category, event.outcome, and event.type.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3300
 - version: "1.4.0"
   changes:
     - description: Update to ECS 8.2

packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -28,6 +28,20 @@ processors:
   - pipeline:
       name: '{{ IngestPipeline "dhcpv6" }}'
       if: "ctx?.log?.file?.path != null && ctx.log.file.path.contains('V6')"
+  - foreach:
+      field: observer.mac
+      ignore_missing: true
+      processor:
+        gsub:
+          field: _ingest._value
+          pattern: '[:]'
+          replacement: '-'
+  - foreach:
+      field: observer.mac
+      ignore_missing: true
+      processor:
+        uppercase:
+          field: _ingest._value
   - remove:
       field: event.original
       if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"

packages/microsoft_dhcp/data_stream/log/fields/ecs.yml

@@ -1,34 +1,40 @@
-- name: ecs.version
-  external: ecs
-- name: message
-  external: ecs
-- name: tags
-  external: ecs
-- name: event.action
-  external: ecs
-- name: event.ingested
-  external: ecs
-- name: event.original
-  external: ecs
-- name: event.kind
-  external: ecs
-- name: event.code
-  external: ecs
-- name: event.timezone
-  external: ecs
-- name: host.ip
-  external: ecs
-- name: host.domain
-  external: ecs
-- name: host.mac
-  external: ecs
-- name: observer.hostname
-  external: ecs
-- name: observer.ip
-  external: ecs
-- name: observer.mac
-  external: ecs
-- name: user.name
-  external: ecs
-- name: log.file.path
-  external: ecs
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: event.action
+- external: ecs
+  name: event.category
+- external: ecs
+  name: event.code
+- external: ecs
+  name: event.ingested
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.original
+- external: ecs
+  name: event.outcome
+- external: ecs
+  name: event.timezone
+- external: ecs
+  name: event.type
+- external: ecs
+  name: host.domain
+- external: ecs
+  name: host.ip
+- external: ecs
+  name: host.mac
+- external: ecs
+  name: log.file.path
+- external: ecs
+  name: message
+- external: ecs
+  name: observer.hostname
+- external: ecs
+  name: observer.ip
+- external: ecs
+  name: observer.mac
+- external: ecs
+  name: tags
+- external: ecs
+  name: user.name

packages/microsoft_dhcp/data_stream/log/sample_event.json

@@ -1,72 +1,77 @@
 {
-    "@timestamp": "2001-01-01T01:01:01.000-05:00",
     "agent": {
-        "ephemeral_id": "71909507-d0ea-484b-a8e4-7a8317aae1a3",
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
         "name": "docker-fleet-agent",
+        "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
         "type": "filebeat",
-        "version": "8.0.0-beta1"
-    },
-    "data_stream": {
-        "dataset": "microsoft_dhcp.log",
-        "namespace": "ep",
-        "type": "logs"
-    },
-    "ecs": {
+        "ephemeral_id": "adc79855-a07e-4f88-b14d-79d03400f73d",
         "version": "8.2.0"
     },
-    "elastic_agent": {
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
-        "snapshot": false,
-        "version": "8.0.0-beta1"
-    },
-    "event": {
-        "action": "dhcp-dns-update",
-        "agent_id_status": "verified",
-        "category": [
-            "network"
-        ],
-        "code": "35",
-        "dataset": "microsoft_dhcp.log",
-        "ingested": "2022-02-03T12:11:29Z",
-        "kind": "event",
-        "original": "35,01/01/01,01:01:01,DNS update request failed,192.168.2.1,host.test.com,000000000000,",
-        "outcome": "failure",
-        "timezone": "America/New_York",
-        "type": [
-            "connection",
-            "denied"
-        ]
-    },
-    "host": {
-        "domain": "host.test.com",
-        "ip": "192.168.2.1",
-        "mac": [
-            "00-00-00-00-00-00"
-        ]
-    },
-    "input": {
-        "type": "log"
-    },
     "log": {
         "file": {
-            "path": "/tmp/service_logs/test-dhcp.log"
+            "path": "/tmp/service_logs/test-dhcpV6.log"
         },
-        "offset": 2407
+        "offset": 1619
+    },
+    "elastic_agent": {
+        "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
+        "version": "8.2.0",
+        "snapshot": false
+    },
+    "message": "DHCPV6 Request",
+    "microsoft": {
+        "dhcp": {
+            "duid": {
+                "length": "18",
+                "hex": "0004A34473BFC27FC55B25E86AF0E1761DAA"
+            }
+        }
     },
-    "message": "DNS update request failed",
+    "tags": [
+        "preserve_original_event",
+        "forwarded",
+        "microsoft_dhcp"
+    ],
     "observer": {
         "hostname": "docker-fleet-agent",
         "ip": [
-            "172.19.0.6"
+            "172.18.0.7"
         ],
         "mac": [
-            "02:42:ac:13:00:06"
+            "02-42-AC-12-00-07"
         ]
     },
-    "tags": [
-        "preserve_original_event",
-        "forwarded",
-        "microsoft_dhcp"
-    ]
+    "input": {
+        "type": "log"
+    },
+    "@timestamp": "2021-12-06T12:43:57.000-05:00",
+    "ecs": {
+        "version": "8.3.0"
+    },
+    "data_stream": {
+        "namespace": "ep",
+        "type": "logs",
+        "dataset": "microsoft_dhcp.log"
+    },
+    "host": {
+        "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6",
+        "domain": "test-host"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "ingested": "2022-05-09T14:40:22Z",
+        "original": "11002,12/06/21,12:43:57,DHCPV6 Request,2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6,test-host,,18,0004A34473BFC27FC55B25E86AF0E1761DAA,,,,,",
+        "code": "11002",
+        "timezone": "America/New_York",
+        "kind": "event",
+        "action": "dhcpv6-request",
+        "category": [
+            "network"
+        ],
+        "type": [
+            "connection",
+            "protocol"
+        ],
+        "dataset": "microsoft_dhcp.log",
+        "outcome": "success"
+    }
 }

packages/microsoft_dhcp/docs/README.md

@@ -20,76 +20,81 @@ An example event for `log` looks as following:
 
 ```json
 {
-    "@timestamp": "2001-01-01T01:01:01.000-05:00",
     "agent": {
-        "ephemeral_id": "71909507-d0ea-484b-a8e4-7a8317aae1a3",
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
         "name": "docker-fleet-agent",
+        "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
         "type": "filebeat",
-        "version": "8.0.0-beta1"
-    },
-    "data_stream": {
-        "dataset": "microsoft_dhcp.log",
-        "namespace": "ep",
-        "type": "logs"
-    },
-    "ecs": {
+        "ephemeral_id": "adc79855-a07e-4f88-b14d-79d03400f73d",
         "version": "8.2.0"
     },
-    "elastic_agent": {
-        "id": "584f3aea-648c-4e58-aba4-32b8f88d4396",
-        "snapshot": false,
-        "version": "8.0.0-beta1"
-    },
-    "event": {
-        "action": "dhcp-dns-update",
-        "agent_id_status": "verified",
-        "category": [
-            "network"
-        ],
-        "code": "35",
-        "dataset": "microsoft_dhcp.log",
-        "ingested": "2022-02-03T12:11:29Z",
-        "kind": "event",
-        "original": "35,01/01/01,01:01:01,DNS update request failed,192.168.2.1,host.test.com,000000000000,",
-        "outcome": "failure",
-        "timezone": "America/New_York",
-        "type": [
-            "connection",
-            "denied"
-        ]
-    },
-    "host": {
-        "domain": "host.test.com",
-        "ip": "192.168.2.1",
-        "mac": [
-            "00-00-00-00-00-00"
-        ]
-    },
-    "input": {
-        "type": "log"
-    },
     "log": {
         "file": {
-            "path": "/tmp/service_logs/test-dhcp.log"
+            "path": "/tmp/service_logs/test-dhcpV6.log"
         },
-        "offset": 2407
+        "offset": 1619
+    },
+    "elastic_agent": {
+        "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478",
+        "version": "8.2.0",
+        "snapshot": false
+    },
+    "message": "DHCPV6 Request",
+    "microsoft": {
+        "dhcp": {
+            "duid": {
+                "length": "18",
+                "hex": "0004A34473BFC27FC55B25E86AF0E1761DAA"
+            }
+        }
     },
-    "message": "DNS update request failed",
+    "tags": [
+        "preserve_original_event",
+        "forwarded",
+        "microsoft_dhcp"
+    ],
     "observer": {
         "hostname": "docker-fleet-agent",
         "ip": [
-            "172.19.0.6"
+            "172.18.0.7"
         ],
         "mac": [
-            "02:42:ac:13:00:06"
+            "02-42-AC-12-00-07"
         ]
     },
-    "tags": [
-        "preserve_original_event",
-        "forwarded",
-        "microsoft_dhcp"
-    ]
+    "input": {
+        "type": "log"
+    },
+    "@timestamp": "2021-12-06T12:43:57.000-05:00",
+    "ecs": {
+        "version": "8.3.0"
+    },
+    "data_stream": {
+        "namespace": "ep",
+        "type": "logs",
+        "dataset": "microsoft_dhcp.log"
+    },
+    "host": {
+        "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6",
+        "domain": "test-host"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "ingested": "2022-05-09T14:40:22Z",
+        "original": "11002,12/06/21,12:43:57,DHCPV6 Request,2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6,test-host,,18,0004A34473BFC27FC55B25E86AF0E1761DAA,,,,,",
+        "code": "11002",
+        "timezone": "America/New_York",
+        "kind": "event",
+        "action": "dhcpv6-request",
+        "category": [
+            "network"
+        ],
+        "type": [
+            "connection",
+            "protocol"
+        ],
+        "dataset": "microsoft_dhcp.log",
+        "outcome": "success"
+    }
 }
 ```
 
@@ -103,13 +108,16 @@ An example event for `log` looks as following:
 | data_stream.type | Data stream type. | constant_keyword |
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword |
+| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword |
 | event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword |
 | event.dataset | Event dataset | constant_keyword |
 | event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred.  It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date |
 | event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword |
 | event.module | Event module | constant_keyword |
 | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword |
+| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword |
 | event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword |
+| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword |
 | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword |
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |