From 0d83068360f143f5c2e6e47ed7017107cabbda63 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Thu, 11 Aug 2022 10:04:14 -0500 Subject: [PATCH 1/3] [zscaler_zia] Remap network.protocol to valid values - Add script processor to build URL for parsing. Script has parameters for valid protocols and a default protocol for a fallback. If network.protocol is valid, then it is used, otherwise the protocol (scheme) used for the URL will be the default (fallback) protocol. The original network.protocol is preserved. - Removed redundant processors - Add test cases - Rename test case files to comply with package spec - Regenerate pipeline test expected files --- packages/zscaler_zia/changelog.yml | 5 + .../web/_dev/test/pipeline/test-web.log | 1 + .../test/pipeline/test-web.log-expected.json | 117 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 39 ++++-- packages/zscaler_zia/manifest.yml | 2 +- 5 files changed, 151 insertions(+), 13 deletions(-) diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml index 2d984299993..5afe1c8ad71 100644 --- a/packages/zscaler_zia/changelog.yml +++ b/packages/zscaler_zia/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.4.1" + changes: + - description: Remap network.protocol to valid values for web data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/9999 # FIXME: Set valid PR - version: "2.4.0" changes: - description: Update package to ECS 8.4.0 diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log index 6abcfff4a70..2dd3bf175a3 100644 --- a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log +++ b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log @@ -3,3 +3,4 @@ { "sourcetype" : "zscalernss-web", "event" :{"time":"Fri Dec 31 07:07:07 2021","login":"test@example.com","proto":"HTTP","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"81.2.69.193","sip":"89.160.20.112","reqmethod":"CONNECT","respcode":"200","ua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} { "sourcetype" : "zscalernss-web", "event" :{"time":"Fri Dec 31 07:07:07 2021","login":"test@example.com","proto":"HTTP","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"555","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"81.2.69.193","sip":"81.2.69.144","reqmethod":"CONNECT","respcode":"200","ua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} { "sourcetype" : "zscalernss-web", "event" :{"time":"Fri Dec 31 07:07:07 2021","login":"test@example.com","proto":"HTTPS","eurl":"www.example.com.com/params?version=10.0.19041.1266&user=65792&Id=1","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"297","respsize":"14135","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Business and Economy","urlcat":"Corporate Marketing","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"81.2.69.193","sip":"81.2.69.143","reqmethod":"GET","respcode":"403","ua":"Microsoft-Delivery-Optimization/10.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Access Blocked","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} +{ "sourcetype" : "zscalernss-web", "event" :{"time":"Thu Aug 29 09:20:35 2022","login":"test@example.com","proto":"HTTP_PROXY","eurl":"www.example.com:443","action":"Allowed","appname":"General Browsing","appclass":"General Browsing","reqsize":"555","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test","dept":"Unknown","cip":"81.2.69.193","sip":"89.160.20.112","reqmethod":"CONNECT","respcode":"200","ua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"None","rulelabel":"None","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json index 8a2172ad011..b4deeaf7c9c 100644 --- a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json +++ b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json @@ -572,6 +572,123 @@ } } } + }, + { + "@timestamp": "2022-08-29T09:20:35.000Z", + "destination": { + "ip": "89.160.20.112" + }, + "ecs": { + "version": "8.4.0" + }, + "event": { + "action": "allowed", + "category": [ + "web" + ], + "kind": "event", + "original": "{ \"sourcetype\" : \"zscalernss-web\", \"event\" :{\"time\":\"Thu Aug 29 09:20:35 2022\",\"login\":\"test@example.com\",\"proto\":\"HTTP_PROXY\",\"eurl\":\"www.example.com:443\",\"action\":\"Allowed\",\"appname\":\"General Browsing\",\"appclass\":\"General Browsing\",\"reqsize\":\"555\",\"respsize\":\"65\",\"stime\":\"0\",\"ctime\":\"0\",\"urlclass\":\"Business Use\",\"urlsupercat\":\"Information Technology\",\"urlcat\":\"Web Search\",\"malwarecat\":\"None\",\"threatname\":\"None\",\"riskscore\":\"0\",\"dlpeng\":\"None\",\"dlpdict\":\"None\",\"location\":\"Test\",\"dept\":\"Unknown\",\"cip\":\"81.2.69.193\",\"sip\":\"89.160.20.112\",\"reqmethod\":\"CONNECT\",\"respcode\":\"200\",\"ua\":\"Windows Microsoft Windows 10 Pro ZTunnel/1.0\",\"ereferer\":\"None\",\"ruletype\":\"None\",\"rulelabel\":\"None\",\"contenttype\":\"Other\",\"unscannabletype\":\"None\",\"deviceowner\":\"administrator1\",\"devicehostname\":\"TestMachine35\"}}", + "risk_score": 0, + "type": [ + "info" + ] + }, + "http": { + "request": { + "bytes": 555, + "method": "CONNECT", + "mime_type": "Other", + "referrer": "None" + }, + "response": { + "bytes": 65, + "status_code": 200 + } + }, + "network": { + "protocol": "http_proxy" + }, + "related": { + "hosts": [ + "TestMachine35" + ], + "ip": [ + "81.2.69.193", + "89.160.20.112" + ] + }, + "rule": { + "name": "None", + "ruleset": "None" + }, + "source": { + "nat": { + "ip": "81.2.69.193" + }, + "user": { + "name": "administrator1" + } + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "www.example.com", + "full": "https://www.example.com:443", + "original": "https://www.example.com:443", + "port": 443, + "scheme": "https" + }, + "user": { + "email": "test@example.com" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "Windows Microsoft Windows 10 Pro ZTunnel/1.0", + "os": { + "full": "Windows 10", + "name": "Windows", + "version": "10" + } + }, + "zscaler_zia": { + "web": { + "app": { + "class": "General Browsing", + "name": "General Browsing" + }, + "ctime": 0, + "department": "Unknown", + "device": { + "hostname": "TestMachine35" + }, + "dpl": { + "dictionaries": "None", + "engine": "None" + }, + "location": "Test", + "malware": { + "category": "None" + }, + "stime": 0, + "threat": { + "name": "None" + }, + "unscannable": { + "type": "None" + }, + "url": { + "category": { + "sub": "Web Search", + "super": "Information Technology" + }, + "class": "Business Use" + } + } + } } ] } \ No newline at end of file diff --git a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml index bf60e4fd845..0474dbab86a 100644 --- a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml @@ -116,13 +116,6 @@ processors: - remove: field: json.respcode ignore_missing: true - - rename: - field: json.proto - target_field: network.protocol - ignore_missing: true - - lowercase: - field: network.protocol - ignore_missing: true - rename: field: json.rulelabel target_field: rule.name @@ -131,11 +124,33 @@ processors: field: json.ruletype target_field: rule.ruleset ignore_missing: true - - set: - if: ctx.network?.protocol != null && ctx.json?.eurl != null - field: json.url - value: "{{{network.protocol}}}://{{{json.eurl}}}" - ignore_failure: true + - rename: + field: json.proto + target_field: network.protocol + ignore_missing: true + - lowercase: + field: network.protocol + ignore_missing: true + - script: + description: Build URI for parsing + tag: Build URI for parsing + lang: painless + params: + valid_protocols: + - http + - https + default_protocol: https + source: | + if (ctx.network?.protocol == null || ctx.json?.eurl == null) { + return; + } + + // Remap network.protocol to a valid value, if necessary. + if (params.valid_protocols.contains(ctx.network.protocol)) { + ctx.json["url"] = ctx.network.protocol + "://" + ctx.json.eurl; + } else { + ctx.json["url"] = params.default_protocol + "://" + ctx.json.eurl; + } - uri_parts: field: json.url on_failure: diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml index a5acb5043f8..1b3e0b22dcc 100644 --- a/packages/zscaler_zia/manifest.yml +++ b/packages/zscaler_zia/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: zscaler_zia title: Zscaler Internet Access -version: 2.4.0 +version: 2.4.1 license: basic description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent. type: integration From 07f11c2abd8ed1627fa34b76ccdedbd4eac1ca53 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Mon, 22 Aug 2022 10:32:31 -0500 Subject: [PATCH 2/3] Update changelog with PR --- packages/zscaler_zia/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml index 5afe1c8ad71..fc35243b00b 100644 --- a/packages/zscaler_zia/changelog.yml +++ b/packages/zscaler_zia/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Remap network.protocol to valid values for web data stream. type: bugfix - link: https://github.com/elastic/integrations/pull/9999 # FIXME: Set valid PR + link: https://github.com/elastic/integrations/pull/4045 - version: "2.4.0" changes: - description: Update package to ECS 8.4.0 From bb13cf97f0bc969a04284e6010ebf7f196557ceb Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 23 Aug 2022 07:56:52 -0500 Subject: [PATCH 3/3] Add note about url parsing to docs --- packages/zscaler_zia/_dev/build/docs/README.md | 4 ++++ packages/zscaler_zia/docs/README.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/zscaler_zia/_dev/build/docs/README.md b/packages/zscaler_zia/_dev/build/docs/README.md index 5874ffddd33..653e19c2c8c 100644 --- a/packages/zscaler_zia/_dev/build/docs/README.md +++ b/packages/zscaler_zia/_dev/build/docs/README.md @@ -157,6 +157,10 @@ Sample Response: { "sourcetype" : "zscalernss-web", "event" :{"time":"Fri Dec 17 07:04:57 2021","login":"test@example.com","proto":"HTTP_PROXY","eurl":"browser.events.data.msn.com:443","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"81.2.69.193","sip":"81.2.69.145","reqmethod":"CONNECT","respcode":"200","ua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} ``` +Caveats: + +- To ensure that URLs are processed correctly, logs which have a `network.protocol` value that is not `http` or `https` will be implicitly converted to `https` for the purposes of URL parsing. The original value of `network.protocol` will be preserved. + ## Fields and Sample event ### Alerts diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md index 5874ffddd33..653e19c2c8c 100644 --- a/packages/zscaler_zia/docs/README.md +++ b/packages/zscaler_zia/docs/README.md @@ -157,6 +157,10 @@ Sample Response: { "sourcetype" : "zscalernss-web", "event" :{"time":"Fri Dec 17 07:04:57 2021","login":"test@example.com","proto":"HTTP_PROXY","eurl":"browser.events.data.msn.com:443","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"81.2.69.193","sip":"81.2.69.145","reqmethod":"CONNECT","respcode":"200","ua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} ``` +Caveats: + +- To ensure that URLs are processed correctly, logs which have a `network.protocol` value that is not `http` or `https` will be implicitly converted to `https` for the purposes of URL parsing. The original value of `network.protocol` will be preserved. + ## Fields and Sample event ### Alerts