diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index cef3558998f..a82207b850e 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.0.1" + changes: + - description: Format client.mac as per ECS. + type: bugfix + link: https://github.com/elastic/integrations/pull/3303 - version: "1.0.0" changes: - description: Add OPNsense support. Add PHP-FPM log parsing. diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json index 1ee6f030b82..5738e4eedf5 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-opensense.log-expected.json @@ -194,7 +194,7 @@ { "@timestamp": "2022-12-31T22:06:16.000-04:00", "client": { - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "destination": { @@ -249,7 +249,7 @@ "port": 67 }, "source": { - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "tags": [ diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json index 0752f7061a5..1183346babd 100644 --- a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-dhcp.log-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2022-07-04T09:39:40.000-04:00", "client": { - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "destination": { @@ -57,7 +57,7 @@ "port": 67 }, "source": { - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "tags": [ @@ -69,7 +69,7 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "destination": { @@ -135,7 +135,7 @@ "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "tags": [ @@ -147,7 +147,7 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "destination": { @@ -219,7 +219,7 @@ "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "tags": [ @@ -231,7 +231,7 @@ "client": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "destination": { @@ -297,7 +297,7 @@ "source": { "address": "10.150.60.56", "ip": "10.150.60.56", - "mac": "4c:55:41:a0:fa:99", + "mac": "4C-55-41-A0-FA-99", "port": 68 }, "tags": [ diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml index 2a604c88d25..f366964fba1 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/dhcp.yml @@ -54,6 +54,14 @@ processors: field: server.ip value: "{{server.address}}" ignore_empty_value: true + - uppercase: + field: client.mac + ignore_missing: true + - gsub: + field: client.mac + pattern: '[:]' + replacement: '-' + ignore_missing: true - set: field: source copy_from: client diff --git a/packages/pfsense/data_stream/log/fields/ecs.yml b/packages/pfsense/data_stream/log/fields/ecs.yml index 51773f7009e..799829902ce 100644 --- a/packages/pfsense/data_stream/log/fields/ecs.yml +++ b/packages/pfsense/data_stream/log/fields/ecs.yml @@ -1,272 +1,268 @@ -- name: '@timestamp' - external: ecs -- name: client.as.number - external: ecs -- name: client.as.organization.name - external: ecs -- name: client.domain - external: ecs -- name: client.geo.city_name - external: ecs -- name: client.geo.country_name - external: ecs -- name: client.geo.country_iso_code - external: ecs -- name: client.geo.continent_name - external: ecs +- external: ecs + name: '@timestamp' +- external: ecs + name: client.address +- external: ecs + name: client.as.number +- external: ecs + name: client.as.organization.name +- external: ecs + name: client.bytes +- external: ecs + name: client.domain +- external: ecs + name: client.geo.city_name +- external: ecs ignore_above: 1024 -- name: client.geo.country_iso_code - external: ecs -- name: client.geo.region_iso_code - external: ecs -- name: client.geo.location - description: Longitude and latitude. + name: client.geo.continent_name +- external: ecs + name: client.geo.country_iso_code +- external: ecs + name: client.geo.country_name +- description: Longitude and latitude. example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: client.geo.location type: geo_point -- name: client.geo.region_name - external: ecs -- name: client.ip - external: ecs -- name: client.address - external: ecs -- name: client.bytes - external: ecs -- name: client.port - external: ecs -- name: client.mac - external: ecs -- name: destination.bytes - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.location - description: Longitude and latitude. +- external: ecs + name: client.geo.region_iso_code +- external: ecs + name: client.geo.region_name +- external: ecs + name: client.ip +- external: ecs + name: client.mac +- external: ecs + name: client.port +- external: ecs + name: destination.address +- external: ecs + name: destination.as.number +- external: ecs + name: destination.as.organization.name +- external: ecs + name: destination.bytes +- external: ecs + name: destination.geo.city_name +- external: ecs + name: destination.geo.continent_name +- external: ecs + name: destination.geo.country_iso_code +- external: ecs + name: destination.geo.country_name +- description: Longitude and latitude. example: '{ "lon": -73.614830, "lat": 45.505918 }' + name: destination.geo.location type: geo_point -- name: destination.geo.name - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs -- name: destination.ip - external: ecs -- name: destination.address - external: ecs -- name: destination.port - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.duration - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.provider - external: ecs -- name: event.reason - external: ecs -- name: event.timezone - external: ecs -- name: event.type - external: ecs -- name: message - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - type: geo_point -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.nat.ip - external: ecs -- name: source.port - external: ecs -- name: source.mac - external: ecs -- name: source.user.id - external: ecs -- name: source.user.full_name - external: ecs -- name: user_agent.device.name - external: ecs -- name: user_agent.name - external: ecs -- name: user_agent.original - external: ecs -- name: user_agent.os.name - external: ecs -- name: user_agent.os.version - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.version - external: ecs -- name: tags - external: ecs -- name: user.domain - external: ecs -- name: user.email - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: user.full_name - external: ecs -- name: url.domain - external: ecs -- name: url.original - external: ecs -- name: url.password - external: ecs -- name: url.port - external: ecs -- name: url.username - external: ecs -- name: url.path - external: ecs -- name: url.query - external: ecs -- name: url.extension - external: ecs -- name: url.scheme - external: ecs -- name: url.full - external: ecs -- name: tls.cipher - external: ecs -- name: tls.version - external: ecs -- name: tls.version_protocol - external: ecs -- name: network.bytes - external: ecs -- name: network.protocol - external: ecs -- name: network.transport - external: ecs -- name: network.community_id - external: ecs -- name: network.iana_number - external: ecs -- name: network.packets - external: ecs -- name: network.type - external: ecs -- name: network.direction - external: ecs -- name: http.response.status_code - external: ecs -- name: http.request.body.bytes - external: ecs -- name: http.response.body.bytes - external: ecs -- name: http.response.bytes - external: ecs -- name: http.request.method - external: ecs -- name: http.request.referrer - external: ecs -- name: http.version - external: ecs -- name: http.response.mime_type - external: ecs -- name: observer.type - external: ecs -- name: observer.vendor - external: ecs -- name: observer.ip - external: ecs -- name: observer.ingress.interface.name - external: ecs -- name: observer.ingress.vlan.id - external: ecs -- name: observer.name - external: ecs -- name: server.address - external: ecs -- name: server.bytes - external: ecs -- name: server.ip - external: ecs -- name: server.port - external: ecs -- name: dns.question.class - external: ecs -- name: dns.question.name - external: ecs -- name: dns.question.registered_domain - external: ecs -- name: dns.question.subdomain - external: ecs -- name: dns.question.top_level_domain - external: ecs -- name: dns.question.type - external: ecs -- name: dns.type - external: ecs -- name: log.level - external: ecs -- name: log.syslog.priority - external: ecs -- name: rule.id - external: ecs -- name: input.type +- external: ecs + name: destination.geo.name +- external: ecs + name: destination.geo.region_iso_code +- external: ecs + name: destination.geo.region_name +- external: ecs + name: destination.ip +- external: ecs + name: destination.port +- external: ecs + name: dns.question.class +- external: ecs + name: dns.question.name +- external: ecs + name: dns.question.registered_domain +- external: ecs + name: dns.question.subdomain +- external: ecs + name: dns.question.top_level_domain +- external: ecs + name: dns.question.type +- external: ecs + name: dns.type +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.action +- external: ecs + name: event.category +- external: ecs + name: event.duration +- external: ecs + name: event.id +- external: ecs + name: event.ingested +- external: ecs + name: event.kind +- external: ecs + name: event.original +- external: ecs + name: event.outcome +- external: ecs + name: event.provider +- external: ecs + name: event.reason +- external: ecs + name: event.timezone +- external: ecs + name: event.type +- external: ecs + name: http.request.body.bytes +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.bytes +- external: ecs + name: http.response.mime_type +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- description: Type of Filebeat input. + name: input.type type: keyword - description: Type of Filebeat input. -- name: process.name - external: ecs -- name: process.pid - external: ecs +- external: ecs + name: log.level +- external: ecs + name: log.syslog.priority +- external: ecs + name: message +- external: ecs + name: network.bytes +- external: ecs + name: network.community_id +- external: ecs + name: network.direction +- external: ecs + name: network.iana_number +- external: ecs + name: network.packets +- external: ecs + name: network.protocol +- external: ecs + name: network.transport +- external: ecs + name: network.type +- external: ecs + name: observer.ingress.interface.name +- external: ecs + name: observer.ingress.vlan.id +- external: ecs + name: observer.ip +- external: ecs + name: observer.name +- external: ecs + name: observer.type +- external: ecs + name: observer.vendor +- external: ecs + name: process.name +- external: ecs + name: process.pid +- external: ecs + name: related.ip +- external: ecs + name: related.user +- external: ecs + name: rule.id +- external: ecs + name: server.address +- external: ecs + name: server.bytes +- external: ecs + name: server.ip +- external: ecs + name: server.port +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.bytes +- external: ecs + name: source.domain +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- external: ecs + name: source.geo.location +- external: ecs + name: source.geo.name +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: source.mac +- external: ecs + name: source.nat.ip +- external: ecs + name: source.port +- external: ecs + name: source.user.full_name +- external: ecs + name: source.user.id +- external: ecs + name: tags +- external: ecs + name: tls.cipher +- external: ecs + name: tls.version +- external: ecs + name: tls.version_protocol +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.full +- external: ecs + name: url.original +- external: ecs + name: url.password +- external: ecs + name: url.path +- external: ecs + name: url.port +- external: ecs + name: url.query +- external: ecs + name: url.scheme +- external: ecs + name: url.username +- external: ecs + name: user.domain +- external: ecs + name: user.email +- external: ecs + name: user.full_name +- external: ecs + name: user.id +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version diff --git a/packages/pfsense/data_stream/log/fields/fields.yml b/packages/pfsense/data_stream/log/fields/fields.yml index f137e647697..7a350eba173 100644 --- a/packages/pfsense/data_stream/log/fields/fields.yml +++ b/packages/pfsense/data_stream/log/fields/fields.yml @@ -78,10 +78,6 @@ type: long description: | ID of the echo request/reply - - name: seq - type: long - description: | - Sequence number of the echo request/reply - name: destination.ip type: ip description: Original destination address of the connection that caused this notification @@ -116,10 +112,6 @@ type: long description: | ICMP code. - - name: id - type: long - description: | - ICMP ID. - name: parameter type: long description: | diff --git a/packages/pfsense/data_stream/log/sample_event.json b/packages/pfsense/data_stream/log/sample_event.json index 5de44aee8db..6f70d5817ed 100644 --- a/packages/pfsense/data_stream/log/sample_event.json +++ b/packages/pfsense/data_stream/log/sample_event.json @@ -5,7 +5,7 @@ "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "pfsense.log", @@ -35,7 +35,7 @@ "elastic_agent": { "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "action": "block", @@ -44,8 +44,7 @@ "network" ], "dataset": "pfsense.log", - "id": "72237", - "ingested": "2022-02-03T09:44:29Z", + "ingested": "2022-05-09T17:35:12Z", "kind": "event", "original": "\u003c134\u003e1 2021-07-03T19:10:14.578288-05:00 pfSense.example.com filterlog 72237 - - 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,32989,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49652,853,0,S,1818117648,,64240,,mss;sackOK;TS;nop;wscale\n", "provider": "filterlog", @@ -61,7 +60,7 @@ }, "log": { "source": { - "address": "172.19.0.7:54953" + "address": "172.18.0.4:58663" }, "syslog": { "priority": 134 @@ -85,7 +84,9 @@ "id": "12" } }, - "name": "pfSense.example.com" + "name": "pfSense.example.com", + "type": "firewall", + "vendor": "netgate" }, "pfsense": { "ip": { @@ -108,6 +109,10 @@ "window": 64240 } }, + "process": { + "name": "filterlog", + "pid": 72237 + }, "related": { "ip": [ "175.16.199.1", diff --git a/packages/pfsense/docs/README.md b/packages/pfsense/docs/README.md index 8c764da26cf..bcdfbd09f3c 100644 --- a/packages/pfsense/docs/README.md +++ b/packages/pfsense/docs/README.md @@ -51,7 +51,7 @@ An example event for `log` looks as following: "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0-beta1" + "version": "8.2.0" }, "data_stream": { "dataset": "pfsense.log", @@ -81,7 +81,7 @@ An example event for `log` looks as following: "elastic_agent": { "id": "584f3aea-648c-4e58-aba4-32b8f88d4396", "snapshot": false, - "version": "8.0.0-beta1" + "version": "8.2.0" }, "event": { "action": "block", @@ -90,8 +90,7 @@ An example event for `log` looks as following: "network" ], "dataset": "pfsense.log", - "id": "72237", - "ingested": "2022-02-03T09:44:29Z", + "ingested": "2022-05-09T17:35:12Z", "kind": "event", "original": "\u003c134\u003e1 2021-07-03T19:10:14.578288-05:00 pfSense.example.com filterlog 72237 - - 146,,,1535324496,igb1.12,match,block,in,4,0x0,,63,32989,0,DF,6,tcp,60,10.170.12.50,175.16.199.1,49652,853,0,S,1818117648,,64240,,mss;sackOK;TS;nop;wscale\n", "provider": "filterlog", @@ -107,7 +106,7 @@ An example event for `log` looks as following: }, "log": { "source": { - "address": "172.19.0.7:54953" + "address": "172.18.0.4:58663" }, "syslog": { "priority": 134 @@ -131,7 +130,9 @@ An example event for `log` looks as following: "id": "12" } }, - "name": "pfSense.example.com" + "name": "pfSense.example.com", + "type": "firewall", + "vendor": "netgate" }, "pfsense": { "ip": { @@ -154,6 +155,10 @@ An example event for `log` looks as following: "window": 64240 } }, + "process": { + "name": "filterlog", + "pid": 72237 + }, "related": { "ip": [ "175.16.199.1", @@ -333,7 +338,7 @@ An example event for `log` looks as following: | pfsense.icmp.parameter | ICMP parameter. | long | | pfsense.icmp.redirect | ICMP redirect address. | ip | | pfsense.icmp.rtime | Receive Timestamp | date | -| pfsense.icmp.seq | Sequence number of the echo request/reply | long | +| pfsense.icmp.seq | ICMP sequence number. | long | | pfsense.icmp.ttime | Transmit Timestamp | date | | pfsense.icmp.type | ICMP type. | keyword | | pfsense.icmp.unreachable.iana_number | Protocol ID number that was unreachable | long | diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 9e4c591c7f9..69b14b9e330 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense Logs -version: 1.0.0 +version: "1.0.1" release: ga description: Collect and parse logs from pfSense and OPNsense devices with Elastic Agent. type: integration