diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b972c9960784..0e9168477567 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add metrics for TCP packet processing. {pull}34333[34333] - Add metrics for unix socket packet processing. {pull}34335[34335] - Add beta `take over` mode for `filestream` for simple migration from `log` inputs {pull}34292[34292] +- Add pagination support for Salesforce module. {issue}34057[34057] {pull}34065[34065] *Auditbeat* diff --git a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml b/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml index ad2833ee848c..5a5af4ab5d35 100644 --- a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml +++ b/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml @@ -30,10 +30,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND CreatedDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND LogDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -41,7 +44,7 @@ chain: replace: $.records[:].Id cursor: last_published_apex: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml b/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml index b1180f270fbf..380c746d7394 100644 --- a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml +++ b/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml @@ -30,10 +30,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND CreatedDate > [[.cursor.last_published_login]] ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND LogDate > [[.cursor.last_published_login]] ORDER BY LogDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -41,7 +44,7 @@ chain: replace: $.records[:].Id cursor: last_published_login: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml index cde7f96128fc..ee0595fdd516 100644 --- a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml +++ b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml @@ -29,10 +29,13 @@ request.proxy_url: {{ .proxy_url }} request.transforms: - set: target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND CreatedDate > [[.cursor.last_published_logout]] ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY LogDate ASC NULLS FIRST" + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST" +response.pagination: + - set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true chain: - step: request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile @@ -40,7 +43,7 @@ chain: replace: $.records[:].Id cursor: last_published_logout: - value: '[[.last_event.CreatedDate]]' + value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml index 9d5e7ebf07b5..34f8f694944a 100644 --- a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml +++ b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml @@ -34,6 +34,11 @@ request.transforms: default: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST" response.split: target: body.records +response.pagination: +- set: + target: url.value + value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' + fail_on_template_error: true cursor: last_published_setupaudittrail: value: '[[.last_event.CreatedDate]]'