From cb420397c48d77c49eb33da061f26dab85d83f72 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 7 Apr 2021 14:58:44 +0200 Subject: [PATCH 1/2] Use fail_on_template_error on google_workspace and okta pagination --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/module/google_workspace/admin/config/config.yml | 1 + x-pack/filebeat/module/google_workspace/drive/config/config.yml | 1 + x-pack/filebeat/module/google_workspace/groups/config/config.yml | 1 + x-pack/filebeat/module/google_workspace/login/config/config.yml | 1 + x-pack/filebeat/module/google_workspace/saml/config/config.yml | 1 + .../module/google_workspace/user_accounts/config/config.yml | 1 + x-pack/filebeat/module/okta/system/config/input.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 90682d343781..ab6e59ee5751 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -388,6 +388,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829] - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] - Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] +- Use fail_on_template_error on google_workspace and okta pagination. {pull}24967[24967] *Heartbeat* diff --git a/x-pack/filebeat/module/google_workspace/admin/config/config.yml b/x-pack/filebeat/module/google_workspace/admin/config/config.yml index ec18b343898a..b49200603527 100644 --- a/x-pack/filebeat/module/google_workspace/admin/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/admin/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/google_workspace/drive/config/config.yml b/x-pack/filebeat/module/google_workspace/drive/config/config.yml index 9f00099c8a64..2d51ee26cda9 100644 --- a/x-pack/filebeat/module/google_workspace/drive/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/drive/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/google_workspace/groups/config/config.yml b/x-pack/filebeat/module/google_workspace/groups/config/config.yml index 261f412a104d..1260cf44f622 100644 --- a/x-pack/filebeat/module/google_workspace/groups/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/groups/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/google_workspace/login/config/config.yml b/x-pack/filebeat/module/google_workspace/login/config/config.yml index 137450f2ad4e..cab5ec5211a9 100644 --- a/x-pack/filebeat/module/google_workspace/login/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/login/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/google_workspace/saml/config/config.yml b/x-pack/filebeat/module/google_workspace/saml/config/config.yml index f8e64ef624b4..17302a6b4bee 100644 --- a/x-pack/filebeat/module/google_workspace/saml/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/saml/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml index 33b6cf987a08..6e5f0141c145 100644 --- a/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/google_workspace/user_accounts/config/config.yml @@ -28,6 +28,7 @@ response.pagination: - set: target: url.params.pageToken value: "[[.last_response.body.nextPageToken]]" + fail_on_template_error: true cursor: last_execution_datetime: value: "[[formatDate now]]" diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index 32f3091a4b84..e40d584df378 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -36,6 +36,7 @@ response.pagination: - set: target: url.value value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' + fail_on_template_error: true cursor: published: From e97208f4beeda13ee048a54b29aaac6d2971392e Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 8 Apr 2021 16:33:37 +0200 Subject: [PATCH 2/2] Change changelog message --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ab6e59ee5751..d06e7fbfd947 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -388,7 +388,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829] - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] - Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] -- Use fail_on_template_error on google_workspace and okta pagination. {pull}24967[24967] +- Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967] *Heartbeat*