From 533302fe95ec5833e038a2936625b6618d003e6d Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Fri, 16 Dec 2022 16:58:06 +0530 Subject: [PATCH 1/2] Add pagination support --- CHANGELOG.next.asciidoc | 1 + .../salesforce/apex-rest/config/apex-rest.yml | 13 ++++++++----- .../salesforce/login-rest/config/login-rest.yml | 13 ++++++++----- .../salesforce/logout-rest/config/logout-rest.yml | 13 ++++++++----- .../config/setupaudittrail-rest.yml | 5 +++++ 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b51e39b68796..6b713707bddb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -182,6 +182,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add metrics for UDP packet processing. {pull}33870[33870] - Convert UDP input to v2 input. {pull}33930[33930] - Improve collection of risk information from Okta debug data. {issue}33677[33677] {pull}34030[34030] +- Add pagination support for Salesforce module. {issue}34057[34057] *Auditbeat* diff --git a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml b/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml index ad2833ee848c..5a5af4ab5d35 100644 --- a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml +++ b/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml @@ -30,10 +30,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND CreatedDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND LogDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -41,7 +44,7 @@ chain: replace: $.records[:].Id cursor: last_published_apex: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml b/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml index b1180f270fbf..380c746d7394 100644 --- a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml +++ b/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml @@ -30,10 +30,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND CreatedDate > [[.cursor.last_published_login]] ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND LogDate > [[.cursor.last_published_login]] ORDER BY LogDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -41,7 +44,7 @@ chain: replace: $.records[:].Id cursor: last_published_login: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml index cde7f96128fc..b5ed3ee515be 100644 --- a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml +++ b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml @@ -29,10 +29,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND CreatedDate > [[.cursor.last_published_logout]] ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY CreatedDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -40,7 +43,7 @@ chain: replace: $.records[:].Id cursor: last_published_logout: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml index 9d5e7ebf07b5..34f8f694944a 100644 --- a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml +++ b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml @@ -34,6 +34,11 @@ request.transforms: default: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST" response.split: target: body.records +response.pagination: +- set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true cursor: last_published_setupaudittrail: value: '[[.last_event.CreatedDate]]' From 8f62da9533ed9cc99c05505880a34186a3b32648 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 17 Jan 2023 18:17:08 +0530 Subject: [PATCH 2/2] Address Review comment --- .../module/salesforce/logout-rest/config/logout-rest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml index b5ed3ee515be..ee0595fdd516 100644 --- a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml +++ b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml @@ -29,7 +29,7 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY CreatedDate ASC NULLS FIRST" + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY LogDate ASC NULLS FIRST" default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST" response.pagination: - set: