Skip to content

Commit

Permalink
Cherry-pick elastic#23424 to 7.x: Sync fixes from Integration Package…
Browse files Browse the repository at this point in the history
… Testing (elastic#24077)

* Sync fixes from Integration Package Testing (elastic#23424)

* Sync changes to AWS CloudTrail

elastic/integrations#408

* Sync changes to CheckPoint Firewall

Change type of event.severity.

elastic/integrations#409

* Sync changes from Cisco ASA / FTD

elastic/integrations#414

* Sync changes from Cisco IOS

Make icmp and igmp fields strings because they are keywords.

elastic/integrations#416

* Sync changes to CrowdStrike Falcon

Fix some field types.

elastic/integrations#377

* Sync changes to Fortinet Firewall

Drop assignip if the value is "N/A".

elastic/integrations#437

* Sync changes to Juniper SRX

Convert event.risk values to float
Protect against missing event.timezone
Convert event.severity to long.

elastic/integrations#443

* Sync changes to Suricata EVE

Convert suricata.eve.flow_id to string because the field is a keyword in the mapping.

elastic/integrations#457

* Sync changes to Zeek DNS

Fix usages of ignore_failure with convert processor.
Make DNS transaction ID a string.

elastic/integrations#448

* Add changelog

(cherry picked from commit bf46572)
  • Loading branch information
andrewkroh authored Feb 17, 2021
1 parent 6806287 commit 94b6732
Show file tree
Hide file tree
Showing 27 changed files with 267 additions and 189 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d

*Filebeat*

- CheckPoint Firewall module: Change event.severity JSON data type to a number because the field mapping is a `long`. {pull}23424[23424]
- Cisco IOS: Change icmp.type/code and igmp.type JSON data types to strings because the fields mappings are `keyword`. {pull}23424[23424]
- CrowdStrike Falcon: Change JSON field types to match the field mappings. {pull}23424[23424]
- Fortinet Firewall: Drop `fortinet.firewall.assignip` when the value is "N/A". {pull}23424[23424]
- Juniper SRX: Change JSON field types to match the field mappings. {pull}23424[23424]
- Suricata EVE: Convert `suricata.eve.flow_id` to string because the field is a keyword in the mapping. {pull}23424[23424]
- Zeek DNS: Ignore failures in data type conversions. And change `dns.id` JSON field to a string to match its `keyword` mapping. {pull}23424[23424]
- Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
- cisco/asa fileset: Fix parsing of 302021 message code. {pull}14519[14519]
- Fix filebeat azure dashboards, event category should be `Alert`. {pull}14668[14668]
Expand Down
8 changes: 4 additions & 4 deletions x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -152,25 +152,25 @@ processors:
Map map = new HashMap();
ctx.aws.cloudtrail.put("flattened", map);
}
if (ctx.json.requestParameters != null) {
if (ctx.json?.requestParameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
}
}
if (ctx.json.responseElements != null) {
if (ctx.json?.responseElements != null) {
ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
}
}
if (ctx.json.additionalEventData != null) {
if (ctx.json?.additionalEventData != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
}
}
if (ctx.json.serviceEventDetails != null) {
if (ctx.json?.serviceEventDetails != null) {
ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,11 @@ processors:
type: long
ignore_failure: true
ignore_missing: true
- rename:
- convert:
field: checkpoint.severity
target_field: event.severity
type: long
ignore_failure: true
ignore_missing: true
- rename:
field: checkpoint.action
Expand Down Expand Up @@ -859,6 +861,7 @@ processors:
- checkpoint.xlatedst
- checkpoint.uid
- checkpoint.time
- checkpoint.severity
- syslog5424_ts
- _temp_
ignore_missing: true
Expand Down
3 changes: 0 additions & 3 deletions x-pack/filebeat/module/cisco/ios/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,6 @@ var ciscoIOS = (function() {
{from: "source.port", type: "long"},
{from: "source.packets", type: "long"},
{from: "source.packets", to: "network.packets", type: "long"},
{from: "icmp.type", type: "long"},
{from: "icmp.code", type: "long"},
{from: "igmp.type", type: "long"},
],
ignore_missing: true,
}).Run;
Expand Down
6 changes: 3 additions & 3 deletions x-pack/filebeat/module/cisco/ios/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ var testCases = []testCase{
"event.outcome": "deny",
"event.severity": int64(6),
"event.type": []string{"connection", "firewall"},
"icmp.code": int64(5),
"icmp.type": int64(3),
"icmp.code": "5",
"icmp.type": "3",
"log.level": "informational",
"log.original": isdef.IsNonEmptyString,
"message": "list 100 denied icmp 198.51.100.1 -> 198.51.100.2 (3/5), 1 packet",
Expand Down Expand Up @@ -117,7 +117,7 @@ var testCases = []testCase{
"event.outcome": "deny",
"event.severity": int64(6),
"event.type": []string{"connection", "firewall"},
"igmp.type": int64(20),
"igmp.type": "20",
"log.level": "informational",
"log.original": isdef.IsNonEmptyString,
"message": "list INBOUND-ON-AP denied igmp 198.51.100.1 -> 224.0.0.2 (20), 1 packet",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
"firewall"
],
"fileset.name": "ios",
"igmp.type": 20,
"igmp.type": "20",
"input.type": "log",
"log.level": "informational",
"log.offset": 140,
Expand Down Expand Up @@ -251,8 +251,8 @@
"firewall"
],
"fileset.name": "ios",
"icmp.code": 4,
"icmp.type": 3,
"icmp.code": "4",
"icmp.type": "3",
"input.type": "log",
"log.level": "informational",
"log.offset": 760,
Expand Down Expand Up @@ -1268,8 +1268,8 @@
"firewall"
],
"fileset.name": "ios",
"icmp.code": 3,
"icmp.type": 3,
"icmp.code": "3",
"icmp.type": "3",
"input.type": "log",
"log.level": "informational",
"log.offset": 4125,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ processors:
# Parse the date included in FTD logs
#
- date:
if: "ctx.event.timezone == null"
if: "ctx.event?.timezone == null && ctx._temp_?.raw_date != null"
field: "_temp_.raw_date"
target_field: "@timestamp"
formats:
Expand Down Expand Up @@ -103,7 +103,7 @@ processors:
},
]
- date:
if: "ctx.event.timezone != null"
if: "ctx.event?.timezone != null && ctx._temp_?.raw_date != null"
timezone: "{{ event.timezone }}"
field: "_temp_.raw_date"
target_field: "@timestamp"
Expand Down
46 changes: 45 additions & 1 deletion x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ var crowdstrikeFalconProcessor = (function () {

function convertToMSEpoch(evt, field) {
var timestamp = evt.Get(field);
if (timestamp == 0) {
evt.Delete(field)
return
}
if (timestamp) {
if (timestamp < 100000000000) { // check if we have a seconds timestamp, this is roughly 1973 in MS
evt.Put(field, timestamp * 1000);
Expand Down Expand Up @@ -103,7 +107,8 @@ var crowdstrikeFalconProcessor = (function () {
type: "ip"
}, {
from: "crowdstrike.event.ProcessId",
to: "process.pid"
to: "process.pid",
type: "long"
}, {
from: "crowdstrike.event.ParentImageFileName",
to: "process.parent.executable"
Expand Down Expand Up @@ -284,6 +289,7 @@ var crowdstrikeFalconProcessor = (function () {
}, {
from: "crowdstrike.event.PID",
to: "process.pid",
type: "long"
},
{
from: "crowdstrike.event.RuleId",
Expand Down Expand Up @@ -421,6 +427,44 @@ var crowdstrikeFalconProcessor = (function () {
ignore_missing: false,
fail_on_error: true
})
.Convert({
fields: [
{
from: "crowdstrike.event.LateralMovement",
type: "long",
},
{
from: "crowdstrike.event.LocalPort",
type: "long",
},
{
from: "crowdstrike.event.MatchCount",
type: "long",
},
{
from: "crowdstrike.event.MatchCountSinceLastReport",
type: "long",
},
{
from: "crowdstrike.event.PID",
type: "long",
},
{
from: "crowdstrike.event.RemotePort",
type: "long",
},
{
from: "source.port",
type: "long",
},
{
from: "destination.port",
type: "long",
}
],
ignore_missing: true,
fail_on_error: false
})
.Build()
.Run
})();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
"crowdstrike.event.PatternDispositionFlags.Rooting": false,
"crowdstrike.event.PatternDispositionFlags.SensorOnly": false,
"crowdstrike.event.PatternDispositionValue": 16,
"crowdstrike.event.ProcessEndTime": 0,
"crowdstrike.event.ProcessId": 38684386611,
"crowdstrike.event.ProcessStartTime": "2018-09-13T13:45:39.000Z",
"crowdstrike.event.SHA256String": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
"crowdstrike.event.HostName": "TESTDEVICE01",
"crowdstrike.event.Ipv": "ipv4",
"crowdstrike.event.LocalAddress": "10.37.60.194",
"crowdstrike.event.LocalPort": "445",
"crowdstrike.event.LocalPort": 445,
"crowdstrike.event.MatchCount": 1,
"crowdstrike.event.MatchCountSinceLastReport": 1,
"crowdstrike.event.NetworkProfile": "2",
"crowdstrike.event.PID": "206158879910",
"crowdstrike.event.PID": 206158879910,
"crowdstrike.event.PolicyID": "74e7f1552a3a4d90a6d65578642c8584",
"crowdstrike.event.PolicyName": "PROD-FW-Workstations-General",
"crowdstrike.event.Protocol": "6",
"crowdstrike.event.RemoteAddress": "10.37.60.21",
"crowdstrike.event.RemotePort": "54952",
"crowdstrike.event.RemotePort": 54952,
"crowdstrike.event.RuleAction": "2",
"crowdstrike.event.RuleFamilyID": "fec73e96a1bf4481be582c3f89b234fa",
"crowdstrike.event.RuleGroupName": "SMB Rules",
Expand All @@ -32,7 +32,7 @@
"crowdstrike.metadata.offset": 70689,
"crowdstrike.metadata.version": "1.0",
"destination.ip": "10.37.60.194",
"destination.port": "445",
"destination.port": 445,
"event.action": "firewall_match_event",
"event.category": [
"network"
Expand All @@ -58,7 +58,7 @@
"message": "Firewall Rule 'Inbound SMB Block & Log Private' triggered",
"network.direction": "ingress",
"network.type": "ipv4",
"process.pid": "206158879910",
"process.pid": 206158879910,
"related.ip": [
"10.37.60.21",
"10.37.60.194"
Expand All @@ -70,7 +70,7 @@
"rule.ruleset": "SMB Rules",
"service.type": "crowdstrike",
"source.ip": "10.37.60.21",
"source.port": "54952",
"source.port": 54952,
"tags": [
"forwarded"
]
Expand Down
45 changes: 31 additions & 14 deletions x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,26 +80,26 @@ processors:
formats:
- UNIX_MS
timezone: "{{fortinet.firewall.tz}}"
if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
- date:
field: fortinet.firewall.eventtime
target_field: event.start
formats:
- UNIX
timezone: "{{fortinet.firewall.tz}}"
if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
- date:
field: fortinet.firewall.eventtime
target_field: event.start
formats:
- UNIX_MS
if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
- date:
field: fortinet.firewall.eventtime
target_field: event.start
formats:
- UNIX
if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
- script:
lang: painless
source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"
Expand Down Expand Up @@ -134,6 +134,9 @@ processors:
field: fortinet.firewall.level
target_field: log.level
ignore_missing: true
- remove:
field: fortinet.firewall.assignip
if: "ctx.fortinet?.firewall?.assignip == 'N/A'"
- remove:
field: fortinet.firewall.dstip
if: "ctx.fortinet?.firewall?.dstip == 'N/A'"
Expand Down Expand Up @@ -222,16 +225,18 @@ processors:
)
- remove:
field:
- _temp
- message
- syslog5424_sd
- syslog5424_pri
- fortinet.firewall.tz
- fortinet.firewall.date
- fortinet.firewall.eventtime
- fortinet.firewall.time
- fortinet.firewall.duration
- host
- _temp.time
- _temp
- message
- syslog5424_sd
- syslog5424_pri
- fortinet.firewall.tz
- fortinet.firewall.date
- fortinet.firewall.devid
- fortinet.firewall.eventtime
- fortinet.firewall.time
- fortinet.firewall.duration
- host
ignore_missing: true
- pipeline:
name: '{< IngestPipeline "event" >}'
Expand All @@ -242,6 +247,18 @@ processors:
- pipeline:
name: '{< IngestPipeline "utm" >}'
if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
- convert:
field: fortinet.firewall.quotamax
type: long
ignore_missing: true
- convert:
field: fortinet.firewall.quotaused
type: long
ignore_missing: true
- convert:
field: fortinet.firewall.size
type: long
ignore_missing: true
on_failure:
- set:
field: error.message
Expand Down
6 changes: 4 additions & 2 deletions x-pack/filebeat/module/juniper/srx/ingest/flow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@ processors:
- append:
field: event.category
value: network
- rename:
- convert:
field: juniper.srx.application_risk
type: float
target_field: event.risk_score
ignore_missing: true
if: "ctx.juniper?.srx?.application_risk != null"
ignore_failure: true
- append:
field: event.type
value:
Expand Down Expand Up @@ -344,6 +345,7 @@ processors:
#############
- remove:
field:
- juniper.srx.application_risk
- juniper.srx.destination_port
- juniper.srx.nat_destination_port
- juniper.srx.bytes_from_client
Expand Down
Loading

0 comments on commit 94b6732

Please sign in to comment.