From 1c9eae291f97c9be77928d17757cb03861dd9d85 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 1 May 2025 11:57:26 +0930 Subject: [PATCH] zscaler_zpa: fix handling of multiple remote IPs, and event categorisation ZScaler appears to sometimes send events with multiple remote IPs. So split the list an raise client.ip to an array if this is the case. The Zscaler ZPA documentation does not make clear what the semantics of this case are, so we bend the ECS a little here. Also fix handling of event.type and event.category to ensure that they agree with ECS definitions, and add event.outcome mappings. --- packages/zscaler_zpa/changelog.yml | 8 ++ .../audit/_dev/test/pipeline/test-audit.log | 1 + .../pipeline/test-audit.log-expected.json | 68 ++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 93 +++++++++++++++---- packages/zscaler_zpa/manifest.yml | 2 +- 5 files changed, 153 insertions(+), 19 deletions(-) diff --git a/packages/zscaler_zpa/changelog.yml b/packages/zscaler_zpa/changelog.yml index 9eafbe83c8f..6c45e39eedf 100644 --- a/packages/zscaler_zpa/changelog.yml +++ b/packages/zscaler_zpa/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "1.22.2" + changes: + - description: Fix handling of remote IP lists in audit data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/13755 + - description: Fix ECS event type, category and outcome mapping of audit events. + type: bugfix + link: https://github.com/elastic/integrations/pull/13755 - version: "1.22.1" changes: - description: Do not set `error.message` for expected behavior related to Zscaler `Host` field. diff --git a/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log b/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log index c6193f87781..3edb0a01bc6 100644 --- a/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log +++ b/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log @@ -1,2 +1,3 @@ {"ModifiedTime":"2021-11-17T04:29:38.000Z","CreationTime":"2021-11-17T04:29:38.000Z","ModifiedBy":12345678901234567,"RequestID":"11111111-1111-1111-1111-111111111111","SessionID":"1idn23nlfm2q1txa5h3r4mep6","AuditOldValue":"","AuditNewValue":"{\"id\":\"72058340288495701\",\"name\":\"Some-Name\",\"domainOrIpAddress\":\"81.2.69.144\",\"description\":\"This is a description field\",\"enabled\":\"true\"}","AuditOperationType":"Create","ObjectType":"Server","ObjectName":"Some-Name","ObjectID":12345678901234567,"CustomerID":98765432109876543,"User":"zpaadmin@xxxxxxxxxxxxxxxxx.zpa-customer.com","ClientAuditUpdate":0} {"ModifiedTime":"","CreationTime":"2021-11-17T04:29:38.000Z","ModifiedBy":12345678901234567,"RequestID":"11111111-1111-1111-1111-111111111111","SessionID":"1idn23nlfm2q1txa5h3r4mep6","AuditOldValue":"","AuditNewValue":"{\"id\":\"72058340288495701\",\"name\":\"Some-Name\",\"domainOrIpAddress\":\"example.com\",\"description\":\"This is a description field\",\"enabled\":\"true\"}","AuditOperationType":"Create","ObjectType":"Server","ObjectName":"Some-Name","ObjectID":12345678901234567,"CustomerID":98765432109876543,"User":"zpaadmin@xxxxxxxxxxxxxxxxx.zpa-customer.com","ClientAuditUpdate":0} +{"ModifiedTime":"","CreationTime":"2025-04-30T16:23:40.000Z","ModifiedBy":288263728720249833,"RequestID":"12d6eccc-718c-4657-b267-83cc1c3f35f6","SessionID":"1samau4fwi7xbsf3317mkd5vz","AuditOldValue":"","AuditNewValue":"{\"loginAttempt\":\"2025-04-30 16:23:40 UTC\",\"remoteIP\":\"81.2.69.142, 81.2.69.144\"}","AuditOperationType":"Sign In","ObjectType":"Authentication","ObjectName":"xxxx","ObjectID":"xxxxx","CustomerID":"xxxxx","User":"xxxx","ClientAuditUpdate":1} diff --git a/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json index 0f3aeb9c74c..058ddde5bcd 100644 --- a/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/zscaler_zpa/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json @@ -13,6 +13,7 @@ "id": "11111111-1111-1111-1111-111111111111", "kind": "event", "original": "{\"ModifiedTime\":\"2021-11-17T04:29:38.000Z\",\"CreationTime\":\"2021-11-17T04:29:38.000Z\",\"ModifiedBy\":12345678901234567,\"RequestID\":\"11111111-1111-1111-1111-111111111111\",\"SessionID\":\"1idn23nlfm2q1txa5h3r4mep6\",\"AuditOldValue\":\"\",\"AuditNewValue\":\"{\\\"id\\\":\\\"72058340288495701\\\",\\\"name\\\":\\\"Some-Name\\\",\\\"domainOrIpAddress\\\":\\\"81.2.69.144\\\",\\\"description\\\":\\\"This is a description field\\\",\\\"enabled\\\":\\\"true\\\"}\",\"AuditOperationType\":\"Create\",\"ObjectType\":\"Server\",\"ObjectName\":\"Some-Name\",\"ObjectID\":12345678901234567,\"CustomerID\":98765432109876543,\"User\":\"zpaadmin@xxxxxxxxxxxxxxxxx.zpa-customer.com\",\"ClientAuditUpdate\":0}", + "outcome": "success", "type": [ "creation" ] @@ -77,6 +78,7 @@ "id": "11111111-1111-1111-1111-111111111111", "kind": "event", "original": "{\"ModifiedTime\":\"\",\"CreationTime\":\"2021-11-17T04:29:38.000Z\",\"ModifiedBy\":12345678901234567,\"RequestID\":\"11111111-1111-1111-1111-111111111111\",\"SessionID\":\"1idn23nlfm2q1txa5h3r4mep6\",\"AuditOldValue\":\"\",\"AuditNewValue\":\"{\\\"id\\\":\\\"72058340288495701\\\",\\\"name\\\":\\\"Some-Name\\\",\\\"domainOrIpAddress\\\":\\\"example.com\\\",\\\"description\\\":\\\"This is a description field\\\",\\\"enabled\\\":\\\"true\\\"}\",\"AuditOperationType\":\"Create\",\"ObjectType\":\"Server\",\"ObjectName\":\"Some-Name\",\"ObjectID\":12345678901234567,\"CustomerID\":98765432109876543,\"User\":\"zpaadmin@xxxxxxxxxxxxxxxxx.zpa-customer.com\",\"ClientAuditUpdate\":0}", + "outcome": "success", "type": [ "creation" ] @@ -123,6 +125,72 @@ } } } + }, + { + "@timestamp": "2025-04-30T16:23:40.000Z", + "client": { + "ip": [ + "81.2.69.142", + "81.2.69.144" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "authentication", + "session" + ], + "created": "2025-04-30T16:23:40.000Z", + "id": "12d6eccc-718c-4657-b267-83cc1c3f35f6", + "kind": "event", + "original": "{\"ModifiedTime\":\"\",\"CreationTime\":\"2025-04-30T16:23:40.000Z\",\"ModifiedBy\":288263728720249833,\"RequestID\":\"12d6eccc-718c-4657-b267-83cc1c3f35f6\",\"SessionID\":\"1samau4fwi7xbsf3317mkd5vz\",\"AuditOldValue\":\"\",\"AuditNewValue\":\"{\\\"loginAttempt\\\":\\\"2025-04-30 16:23:40 UTC\\\",\\\"remoteIP\\\":\\\"81.2.69.142, 81.2.69.144\\\"}\",\"AuditOperationType\":\"Sign In\",\"ObjectType\":\"Authentication\",\"ObjectName\":\"xxxx\",\"ObjectID\":\"xxxxx\",\"CustomerID\":\"xxxxx\",\"User\":\"xxxx\",\"ClientAuditUpdate\":1}", + "outcome": "success", + "type": [ + "start" + ] + }, + "organization": { + "id": "xxxxx" + }, + "related": { + "ip": [ + "81.2.69.142", + "81.2.69.144" + ], + "user": [ + "288263728720249833", + "xxxx" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "id": "288263728720249833", + "name": "xxxx" + }, + "zscaler_zpa": { + "audit": { + "client_audit_update": 1, + "object": { + "id": "xxxxx", + "name": "xxxx", + "type": "Authentication" + }, + "operation_type": "Sign In", + "session": { + "id": "1samau4fwi7xbsf3317mkd5vz" + }, + "value": { + "new": { + "loginAttempt": "2025-04-30 16:23:40 UTC", + "remoteIP": "81.2.69.142, 81.2.69.144" + } + } + } + } } ] } diff --git a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 7315d7d54be..5469e2f3745 100644 --- a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -39,30 +39,74 @@ processors: field: event.created copy_from: '@timestamp' ignore_failure: true - - append: - field: event.category - value: iam - set: field: event.kind value: event - script: if: ctx.json?.AuditOperationType != null && ctx.json.AuditOperationType != '' lang: painless + params: + event_classification: + 'create': + type: + - creation + category: + - iam + outcome: success + 'delete': + type: + - deletion + category: + - iam + outcome: success + 'update': + type: + - change + category: + - iam + outcome: success + 'sign in': + type: + - start + category: + - authentication + - session + outcome: success + 'sign in failure': + type: + - start + - error + category: failure + - session + outcome: + 'download': + type: + - info + - access + outcome: success + category: + - file + 'sign out': + type: + - end + category: + - session + outcome: success + 'client session revoked': + type: + - change + - deletion + category: + - iam + outcome: success source: | - def eventType = ctx.json.AuditOperationType?.toLowerCase(); - ctx.event.type = new ArrayList(); - Map referenceTable = [ - 'create': ['creation'], - 'delete': ['deletion'], - 'update': ['change'], - 'sign in': ['access', 'allowed'], - 'sign in failure': ['access', 'error'], - 'download': ['info'], - 'sign out': ['access'], - 'client session revoked': ['end'] - ]; - - ctx.event.type = referenceTable[eventType]; + def class = params.event_classification[ctx.json.AuditOperationType?.toLowerCase()]; + if (class == null) { + return; + } + ctx.event.type = class.type; + ctx.event.category = class.category; + ctx.event.outcome = class.outcome; - rename: field: json.RequestID target_field: event.id @@ -216,12 +260,25 @@ processors: ctx.server = new HashMap(); ctx.server.address = valuesMap?.domainOrIpAddress; } + - split: + field: client.ip + separator: ', *' + if: ctx.client?.ip instanceof String && ctx.client.ip.contains(',') - append: field: related.ip value: '{{{client.ip}}}' - if: ctx.client?.ip != null + if: ctx.client?.ip instanceof String allow_duplicates: false ignore_failure: true + - foreach: + field: client.ip + if: ctx.client?.ip instanceof List + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false + ignore_failure: true - convert: field: server.address target_field: server.ip diff --git a/packages/zscaler_zpa/manifest.yml b/packages/zscaler_zpa/manifest.yml index 5bd5aa2697e..763ecbef07f 100644 --- a/packages/zscaler_zpa/manifest.yml +++ b/packages/zscaler_zpa/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: zscaler_zpa title: Zscaler Private Access -version: "1.22.1" +version: "1.22.2" source: license: Elastic-2.0 description: Collect logs from Zscaler Private Access (ZPA) with Elastic Agent.