From 5625d41126b96fb19f870ec5e14a7c37e83fca00 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:27:12 -0500 Subject: [PATCH 01/21] Change host.domain to host.name Fix improper field name --- .../data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index 6f891b51df0..1ddad44a95c 100644 --- a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -10,7 +10,7 @@ processors: - _tmp_.time - message - host.ip - - host.domain + - host.name - _tmp_.mac - user.name - microsoft.dhcp.transaction_id From 24d98054c93955723a79101f4b19b72030517d6e Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:27:40 -0500 Subject: [PATCH 02/21] Fix improper field name --- .../data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml index 7c808cad13d..ecff74574b4 100644 --- a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml +++ b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml @@ -9,7 +9,7 @@ processors: - _tmp_.time - message - host.ip - - host.domain + - host.name - microsoft.dhcp.error_code - microsoft.dhcp.duid.length - microsoft.dhcp.duid.hex From 4b6a2dc5f5d026edc764de12e5dfbe8ee43f831f Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:29:13 -0500 Subject: [PATCH 03/21] Update field name in sample --- packages/microsoft_dhcp/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index 4a669c13707..db4f42083bb 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -75,7 +75,7 @@ An example event for `log` looks as following: }, "host": { "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "domain": "test-host" + "name": "test-host" }, "event": { "agent_id_status": "verified", From e7c13a9090b176fc1003d08ec7575a0dd4bae1db Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:32:33 -0500 Subject: [PATCH 04/21] Update documentation and ECS field --- packages/microsoft_dhcp/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index db4f42083bb..d4c32c103c8 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -118,7 +118,7 @@ An example event for `log` looks as following: | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | | event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.ip | Host ip addresses. | ip | | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | input.type | | keyword | From c22fbc0d234915143b39f2438396f12c01ea1942 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:36:43 -0500 Subject: [PATCH 05/21] Update changelog.yml --- packages/microsoft_dhcp/changelog.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml index f88b038889c..4318722fe08 100644 --- a/packages/microsoft_dhcp/changelog.yml +++ b/packages/microsoft_dhcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.6.1" + changes: + - description: Change host.domain to host.name to reflect the event data + type: enhancement + link: https://github.com/elastic/integrations/pull/42** - version: "1.6.0" changes: - description: Update package to ECS 8.4.0 From eb171996e6492d1ed96a747b2b64d0e38ae99feb Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:37:47 -0500 Subject: [PATCH 06/21] Change host.domain to host.name --- packages/microsoft_dhcp/data_stream/log/fields/ecs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml index dc0d37a45b8..6272faf65ab 100644 --- a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml +++ b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml @@ -19,7 +19,7 @@ - external: ecs name: event.type - external: ecs - name: host.domain + name: host.name - external: ecs name: host.ip - external: ecs From 05b68f5a6fe1368b9f10cbcad8b3d36e1e6445d9 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:39:59 -0500 Subject: [PATCH 07/21] Update host.domain to host.name --- .../log/_dev/test/pipeline/test-log.log-expected.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 28b8eaa563f..2a02acfe039 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -118,7 +118,7 @@ ] }, "host": { - "domain": "1-07.test.com", + "name": "1-07.test.com", "ip": "172.28.53.173" }, "log": { @@ -640,4 +640,4 @@ ] } ] -} \ No newline at end of file +} From 3eb8ef65058c0d11284d0d2ec9421c11f324b60d Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:42:27 -0500 Subject: [PATCH 08/21] Change host.domain and to host.name --- .../pipeline/test-logv6.log-expected.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json index 6664a0e5868..9b621eca0fe 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json @@ -122,7 +122,7 @@ "timezone": "America/New_York" }, "host": { - "domain": "test.local" + "name": "test.local" }, "log": { "file": { @@ -155,7 +155,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -197,7 +197,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -239,7 +239,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -281,7 +281,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -323,7 +323,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -365,7 +365,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -407,7 +407,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -449,7 +449,7 @@ ] }, "host": { - "domain": "test-host", + "name": "test-host", "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "log": { @@ -500,4 +500,4 @@ ] } ] -} \ No newline at end of file +} From 4fa634f0ea2fd235e7909e9cba4a3d519892b4b2 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:44:26 -0500 Subject: [PATCH 09/21] Change host.domain to host.name --- .../test/pipeline/test-log.log-expected.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 2a02acfe039..0c8863b6dca 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -78,7 +78,7 @@ ] }, "host": { - "domain": "057182593757.test.com", + "name": "057182593757.test.com", "ip": "172.28.43.169" }, "log": { @@ -158,7 +158,7 @@ ] }, "host": { - "domain": "3-07.test.com", + "name": "3-07.test.com", "ip": "172.28.53.36" }, "log": { @@ -240,7 +240,7 @@ ] }, "host": { - "domain": "035856103966.test.com", + "name": "035856103966.test.com", "ip": "172.28.43.159" }, "log": { @@ -280,7 +280,7 @@ ] }, "host": { - "domain": "001100581357.test.com", + "name": "001100581357.test.com", "ip": "172.28.40.35" }, "log": { @@ -321,7 +321,7 @@ ] }, "host": { - "domain": "host.test.com", + "name": "host.test.com", "ip": "192.168.2.1", "mac": [ "00-00-00-00-00-00" @@ -358,7 +358,7 @@ ] }, "host": { - "domain": "host.test.com", + "name": "host.test.com", "ip": "192.168.2.10", "mac": [ "00-00-00-00-00-00" @@ -401,7 +401,7 @@ ] }, "host": { - "domain": "host.test.com", + "name": "host.test.com", "ip": "192.168.2.20", "mac": [ "00-00-00-00-00-00" @@ -484,7 +484,7 @@ ] }, "host": { - "domain": "hostname.test.com", + "name": "hostname.test.com", "ip": "10.10.10.10" }, "log": { @@ -564,7 +564,7 @@ ] }, "host": { - "domain": "domain.local" + "name": "domain.local" }, "log": { "file": { @@ -597,7 +597,7 @@ ] }, "host": { - "domain": "domain.local" + "name": "domain.local" }, "log": { "file": { From ffbf587e014d86d943c9943e44a256954186cb32 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:52:12 -0500 Subject: [PATCH 10/21] Update changelog.yml --- packages/microsoft_dhcp/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml index 4318722fe08..fb0020ecbac 100644 --- a/packages/microsoft_dhcp/changelog.yml +++ b/packages/microsoft_dhcp/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Change host.domain to host.name to reflect the event data type: enhancement - link: https://github.com/elastic/integrations/pull/42** + link: https://github.com/elastic/integrations/pull/4279 - version: "1.6.0" changes: - description: Update package to ECS 8.4.0 From 4384ed938c2b96839ab98a3b03007ecc0b58f9d6 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 14:54:22 -0500 Subject: [PATCH 11/21] Update changelog.yml --- packages/microsoft_dhcp/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml index fb0020ecbac..2dfeb8b5ef1 100644 --- a/packages/microsoft_dhcp/changelog.yml +++ b/packages/microsoft_dhcp/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Change host.domain to host.name to reflect the event data type: enhancement - link: https://github.com/elastic/integrations/pull/4279 + link: https://github.com/elastic/integrations/pull/4280 - version: "1.6.0" changes: - description: Update package to ECS 8.4.0 From e128baf04e33f5aa85f40871129f7489f6304fd7 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:36:21 -0500 Subject: [PATCH 12/21] Update changelog.yml --- packages/microsoft_dhcp/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microsoft_dhcp/changelog.yml b/packages/microsoft_dhcp/changelog.yml index 2dfeb8b5ef1..e98e18db889 100644 --- a/packages/microsoft_dhcp/changelog.yml +++ b/packages/microsoft_dhcp/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top -- version: "1.6.1" +- version: "1.7.0" changes: - - description: Change host.domain to host.name to reflect the event data + - description: Change host.domain to host.name to reflect the event data and then extract host.domain from the host.name type: enhancement link: https://github.com/elastic/integrations/pull/4280 - version: "1.6.0" From f242e24025be433c2c0287543abab4cf87596dce Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:47:48 -0500 Subject: [PATCH 13/21] Add host.domain grok extraction from host.name --- .../log/elasticsearch/ingest_pipeline/dhcp.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index 1ddad44a95c..e2ad56f4133 100644 --- a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -25,6 +25,14 @@ processors: - microsoft.dhcp.relay_agent_info - microsoft.dhcp.dns_error_code ignore_failure: true + - grok: + field: host.name + if: 'ctx.host?.name != null && ctx.host.name.contains(".")' + patterns: + - "%{HOSTNAME}\\.%{GREEDYDATA:host.domain}" + pattern_definitions: + "HOSTNAME": "[^.]+" + ignore_failure: true - set: field: _tmp_.timestamp value: "{{{_tmp_.date}}} {{{_tmp_.time}}}" From 4c2c8a67b1bba297b3bc0a432aa314d969ec442f Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:50:43 -0500 Subject: [PATCH 14/21] Add host.domain grok extraction from host.name --- .../log/elasticsearch/ingest_pipeline/dhcpv6.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml index ecff74574b4..ff848a28a8b 100644 --- a/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml +++ b/packages/microsoft_dhcp/data_stream/log/elasticsearch/ingest_pipeline/dhcpv6.yml @@ -17,6 +17,14 @@ processors: - microsoft.dhcp.dhc_id - microsoft.dhcp.subnet_prefix ignore_failure: true + - grok: + field: host.name + if: 'ctx.host?.name != null && ctx.host.name.contains(".")' + patterns: + - "%{HOSTNAME}\\.%{GREEDYDATA:host.domain}" + pattern_definitions: + "HOSTNAME": "[^.]+" + ignore_failure: true - set: field: _tmp_.timestamp value: "{{{_tmp_.date}}} {{{_tmp_.time}}}" From 2e09bb5ba69d4cb111da92829814051f62100108 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:54:49 -0500 Subject: [PATCH 15/21] Examples of host.domain extracted from host.name --- .../log/_dev/test/pipeline/test-log.log-expected.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 0c8863b6dca..170a5ae3cdf 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -78,6 +78,7 @@ ] }, "host": { + "domain" : "test.com", "name": "057182593757.test.com", "ip": "172.28.43.169" }, @@ -118,6 +119,7 @@ ] }, "host": { + "domain" : "test.com", "name": "1-07.test.com", "ip": "172.28.53.173" }, @@ -158,6 +160,7 @@ ] }, "host": { + "domain" : "test.com", "name": "3-07.test.com", "ip": "172.28.53.36" }, @@ -240,6 +243,7 @@ ] }, "host": { + "domain" : "test.com", "name": "035856103966.test.com", "ip": "172.28.43.159" }, @@ -280,6 +284,7 @@ ] }, "host": { + "domain" : "test.com", "name": "001100581357.test.com", "ip": "172.28.40.35" }, @@ -321,6 +326,7 @@ ] }, "host": { + "domain" : "test.com", "name": "host.test.com", "ip": "192.168.2.1", "mac": [ @@ -358,6 +364,7 @@ ] }, "host": { + "domain" : "test.com", "name": "host.test.com", "ip": "192.168.2.10", "mac": [ @@ -401,6 +408,7 @@ ] }, "host": { + "domain" : "test.com", "name": "host.test.com", "ip": "192.168.2.20", "mac": [ @@ -484,6 +492,7 @@ ] }, "host": { + "domain" : "test.com", "name": "hostname.test.com", "ip": "10.10.10.10" }, @@ -564,6 +573,7 @@ ] }, "host": { + "domain" : "local", "name": "domain.local" }, "log": { @@ -597,6 +607,7 @@ ] }, "host": { + "domain" : "local", "name": "domain.local" }, "log": { From d8942bbc64d73c3a3fd51501da7bcbccb0368d93 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:56:52 -0500 Subject: [PATCH 16/21] Add host.domain example --- .../log/_dev/test/pipeline/test-logv6.log-expected.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json index 9b621eca0fe..842e819cc53 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json @@ -122,6 +122,7 @@ "timezone": "America/New_York" }, "host": { + "domain" : "local", "name": "test.local" }, "log": { From 2676208487a5a5bb4f06501ea7685aca1ccd0d80 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 18:59:06 -0500 Subject: [PATCH 17/21] Add host.domain back --- packages/microsoft_dhcp/data_stream/log/fields/ecs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml index 6272faf65ab..fa4d81c3383 100644 --- a/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml +++ b/packages/microsoft_dhcp/data_stream/log/fields/ecs.yml @@ -18,6 +18,8 @@ name: event.timezone - external: ecs name: event.type +- external: ecs + name: host.domain - external: ecs name: host.name - external: ecs From cc74f5231c5f9eb95bf17cccb211380dcd84abca Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 22 Sep 2022 19:00:36 -0500 Subject: [PATCH 18/21] Add host.domain back in --- packages/microsoft_dhcp/docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index d4c32c103c8..dfdc1430875 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -118,6 +118,7 @@ An example event for `log` looks as following: | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | | event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.ip | Host ip addresses. | ip | | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | From b501ddc37f114a3ff80b8847a0cb5e500d1b0572 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 24 Sep 2022 09:17:09 -0500 Subject: [PATCH 19/21] Update version info to latest --- packages/microsoft_dhcp/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/manifest.yml b/packages/microsoft_dhcp/manifest.yml index 8c749bcaf3a..d9135a0745d 100644 --- a/packages/microsoft_dhcp/manifest.yml +++ b/packages/microsoft_dhcp/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: microsoft_dhcp title: Microsoft DHCP -version: "1.6.0" +version: "1.7.0" license: basic description: Collect logs from Microsoft DHCP with Elastic Agent. type: integration From 33f65b062922bde506c4ac81c377771b10aa1ab3 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 24 Sep 2022 20:42:37 -0500 Subject: [PATCH 20/21] Add host.domain example --- packages/microsoft_dhcp/docs/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index dfdc1430875..359dea668dc 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -75,7 +75,8 @@ An example event for `log` looks as following: }, "host": { "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "name": "test-host" + "name": "test-host.local", + "domain": "local" }, "event": { "agent_id_status": "verified", From 814c30879db78800a1636b0a7596ee7d4d2a4e3b Mon Sep 17 00:00:00 2001 From: Nicholas Penning Date: Mon, 26 Sep 2022 06:17:23 -0500 Subject: [PATCH 21/21] Used purposed patch to update the PR. --- .../test/pipeline/test-log.log-expected.json | 60 ++++----- .../pipeline/test-logv6.log-expected.json | 36 +++--- .../data_stream/log/sample_event.json | 114 ++++++++--------- packages/microsoft_dhcp/docs/README.md | 117 +++++++++--------- 4 files changed, 159 insertions(+), 168 deletions(-) diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 170a5ae3cdf..60928c9bd58 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -78,9 +78,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "057182593757.test.com", - "ip": "172.28.43.169" + "domain": "test.com", + "ip": "172.28.43.169", + "name": "057182593757.test.com" }, "log": { "file": { @@ -119,9 +119,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "1-07.test.com", - "ip": "172.28.53.173" + "domain": "test.com", + "ip": "172.28.53.173", + "name": "1-07.test.com" }, "log": { "file": { @@ -160,9 +160,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "3-07.test.com", - "ip": "172.28.53.36" + "domain": "test.com", + "ip": "172.28.53.36", + "name": "3-07.test.com" }, "log": { "file": { @@ -243,9 +243,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "035856103966.test.com", - "ip": "172.28.43.159" + "domain": "test.com", + "ip": "172.28.43.159", + "name": "035856103966.test.com" }, "log": { "file": { @@ -284,9 +284,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "001100581357.test.com", - "ip": "172.28.40.35" + "domain": "test.com", + "ip": "172.28.40.35", + "name": "001100581357.test.com" }, "log": { "file": { @@ -326,12 +326,12 @@ ] }, "host": { - "domain" : "test.com", - "name": "host.test.com", + "domain": "test.com", "ip": "192.168.2.1", "mac": [ "00-00-00-00-00-00" - ] + ], + "name": "host.test.com" }, "log": { "file": { @@ -364,12 +364,12 @@ ] }, "host": { - "domain" : "test.com", - "name": "host.test.com", + "domain": "test.com", "ip": "192.168.2.10", "mac": [ "00-00-00-00-00-00" - ] + ], + "name": "host.test.com" }, "log": { "file": { @@ -408,12 +408,12 @@ ] }, "host": { - "domain" : "test.com", - "name": "host.test.com", + "domain": "test.com", "ip": "192.168.2.20", "mac": [ "00-00-00-00-00-00" - ] + ], + "name": "host.test.com" }, "log": { "file": { @@ -492,9 +492,9 @@ ] }, "host": { - "domain" : "test.com", - "name": "hostname.test.com", - "ip": "10.10.10.10" + "domain": "test.com", + "ip": "10.10.10.10", + "name": "hostname.test.com" }, "log": { "file": { @@ -573,7 +573,7 @@ ] }, "host": { - "domain" : "local", + "domain": "local", "name": "domain.local" }, "log": { @@ -607,7 +607,7 @@ ] }, "host": { - "domain" : "local", + "domain": "local", "name": "domain.local" }, "log": { @@ -651,4 +651,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json index 842e819cc53..5e26c6d2750 100644 --- a/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json +++ b/packages/microsoft_dhcp/data_stream/log/_dev/test/pipeline/test-logv6.log-expected.json @@ -122,7 +122,7 @@ "timezone": "America/New_York" }, "host": { - "domain" : "local", + "domain": "local", "name": "test.local" }, "log": { @@ -156,8 +156,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -198,8 +198,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -240,8 +240,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -282,8 +282,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -324,8 +324,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -366,8 +366,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -408,8 +408,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -450,8 +450,8 @@ ] }, "host": { - "name": "test-host", - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "name": "test-host" }, "log": { "file": { @@ -501,4 +501,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/packages/microsoft_dhcp/data_stream/log/sample_event.json b/packages/microsoft_dhcp/data_stream/log/sample_event.json index fc2dbc65249..c56a4fede80 100644 --- a/packages/microsoft_dhcp/data_stream/log/sample_event.json +++ b/packages/microsoft_dhcp/data_stream/log/sample_event.json @@ -1,77 +1,73 @@ { + "@timestamp": "2001-01-01T01:01:01.000-05:00", "agent": { + "ephemeral_id": "268da6cf-879e-4478-b666-96c44fba3109", + "id": "3355f43c-4984-4a3f-b93d-56ac8c865c8c", "name": "docker-fleet-agent", - "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "type": "filebeat", - "ephemeral_id": "adc79855-a07e-4f88-b14d-79d03400f73d", - "version": "8.2.0" - }, - "log": { - "file": { - "path": "/tmp/service_logs/test-dhcpV6.log" - }, - "offset": 1619 - }, - "elastic_agent": { - "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", - "version": "8.2.0", - "snapshot": false - }, - "message": "DHCPV6 Request", - "microsoft": { - "dhcp": { - "duid": { - "length": "18", - "hex": "0004A34473BFC27FC55B25E86AF0E1761DAA" - } - } - }, - "tags": [ - "preserve_original_event", - "forwarded", - "microsoft_dhcp" - ], - "observer": { - "hostname": "docker-fleet-agent", - "ip": [ - "172.18.0.7" - ], - "mac": [ - "02-42-AC-12-00-07" - ] - }, - "input": { - "type": "log" - }, - "@timestamp": "2021-12-06T12:43:57.000-05:00", - "ecs": { - "version": "8.3.0" + "version": "8.4.1" }, "data_stream": { + "dataset": "microsoft_dhcp.log", "namespace": "ep", - "type": "logs", - "dataset": "microsoft_dhcp.log" + "type": "logs" }, - "host": { - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "domain": "test-host" + "ecs": { + "version": "8.4.0" + }, + "elastic_agent": { + "id": "3355f43c-4984-4a3f-b93d-56ac8c865c8c", + "snapshot": false, + "version": "8.4.1" }, "event": { + "action": "dhcp-dns-update", "agent_id_status": "verified", - "ingested": "2022-05-09T14:40:22Z", - "original": "11002,12/06/21,12:43:57,DHCPV6 Request,2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6,test-host,,18,0004A34473BFC27FC55B25E86AF0E1761DAA,,,,,", - "code": "11002", - "timezone": "America/New_York", - "kind": "event", - "action": "dhcpv6-request", "category": [ "network" ], + "code": "35", + "dataset": "microsoft_dhcp.log", + "ingested": "2022-09-26T06:06:08Z", + "kind": "event", + "original": "35,01/01/01,01:01:01,DNS update request failed,192.168.2.1,host.test.com,000000000000,", + "outcome": "failure", + "timezone": "America/New_York", "type": [ "connection", - "protocol" + "denied" + ] + }, + "host": { + "domain": "test.com", + "ip": "192.168.2.1", + "mac": [ + "00-00-00-00-00-00" ], - "dataset": "microsoft_dhcp.log", - "outcome": "success" - } + "name": "host.test.com" + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/test-dhcp.log" + }, + "offset": 2407 + }, + "message": "DNS update request failed", + "observer": { + "hostname": "docker-fleet-agent", + "ip": [ + "172.23.0.4" + ], + "mac": [ + "02-42-AC-17-00-04" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "microsoft_dhcp" + ] } \ No newline at end of file diff --git a/packages/microsoft_dhcp/docs/README.md b/packages/microsoft_dhcp/docs/README.md index 359dea668dc..d840ba02d05 100644 --- a/packages/microsoft_dhcp/docs/README.md +++ b/packages/microsoft_dhcp/docs/README.md @@ -20,82 +20,77 @@ An example event for `log` looks as following: ```json { + "@timestamp": "2001-01-01T01:01:01.000-05:00", "agent": { + "ephemeral_id": "268da6cf-879e-4478-b666-96c44fba3109", + "id": "3355f43c-4984-4a3f-b93d-56ac8c865c8c", "name": "docker-fleet-agent", - "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", "type": "filebeat", - "ephemeral_id": "adc79855-a07e-4f88-b14d-79d03400f73d", - "version": "8.2.0" - }, - "log": { - "file": { - "path": "/tmp/service_logs/test-dhcpV6.log" - }, - "offset": 1619 - }, - "elastic_agent": { - "id": "ca0beb8d-9522-4450-8af7-3cb7f3d8c478", - "version": "8.2.0", - "snapshot": false - }, - "message": "DHCPV6 Request", - "microsoft": { - "dhcp": { - "duid": { - "length": "18", - "hex": "0004A34473BFC27FC55B25E86AF0E1761DAA" - } - } - }, - "tags": [ - "preserve_original_event", - "forwarded", - "microsoft_dhcp" - ], - "observer": { - "hostname": "docker-fleet-agent", - "ip": [ - "172.18.0.7" - ], - "mac": [ - "02-42-AC-12-00-07" - ] - }, - "input": { - "type": "log" - }, - "@timestamp": "2021-12-06T12:43:57.000-05:00", - "ecs": { - "version": "8.3.0" + "version": "8.4.1" }, "data_stream": { + "dataset": "microsoft_dhcp.log", "namespace": "ep", - "type": "logs", - "dataset": "microsoft_dhcp.log" + "type": "logs" }, - "host": { - "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", - "name": "test-host.local", - "domain": "local" + "ecs": { + "version": "8.4.0" + }, + "elastic_agent": { + "id": "3355f43c-4984-4a3f-b93d-56ac8c865c8c", + "snapshot": false, + "version": "8.4.1" }, "event": { + "action": "dhcp-dns-update", "agent_id_status": "verified", - "ingested": "2022-05-09T14:40:22Z", - "original": "11002,12/06/21,12:43:57,DHCPV6 Request,2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6,test-host,,18,0004A34473BFC27FC55B25E86AF0E1761DAA,,,,,", - "code": "11002", - "timezone": "America/New_York", - "kind": "event", - "action": "dhcpv6-request", "category": [ "network" ], + "code": "35", + "dataset": "microsoft_dhcp.log", + "ingested": "2022-09-26T06:06:08Z", + "kind": "event", + "original": "35,01/01/01,01:01:01,DNS update request failed,192.168.2.1,host.test.com,000000000000,", + "outcome": "failure", + "timezone": "America/New_York", "type": [ "connection", - "protocol" + "denied" + ] + }, + "host": { + "domain": "test.com", + "ip": "192.168.2.1", + "mac": [ + "00-00-00-00-00-00" ], - "dataset": "microsoft_dhcp.log", - "outcome": "success" - } + "name": "host.test.com" + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/test-dhcp.log" + }, + "offset": 2407 + }, + "message": "DNS update request failed", + "observer": { + "hostname": "docker-fleet-agent", + "ip": [ + "172.23.0.4" + ], + "mac": [ + "02-42-AC-17-00-04" + ] + }, + "tags": [ + "preserve_original_event", + "forwarded", + "microsoft_dhcp" + ] } ``` @@ -120,9 +115,9 @@ An example event for `log` looks as following: | event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | | event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.ip | Host ip addresses. | ip | | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | input.type | | keyword | | log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | | long |