From 9c124f898061fb151fcf4dcfa3cdda9a7560ce9a Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 22 May 2025 13:38:39 +0930 Subject: [PATCH] github: squelch errors from pagination ends --- packages/github/changelog.yml | 13 +++++++++---- .../data_stream/audit/agent/stream/httpjson.yml.hbs | 1 + .../code_scanning/agent/stream/httpjson.yml.hbs | 1 + .../dependabot/agent/stream/httpjson.yml.hbs | 4 ++++ .../issues/agent/stream/httpjson.yml.hbs | 1 + .../secret_scanning/agent/stream/httpjson.yml.hbs | 1 + packages/github/manifest.yml | 4 ++-- 7 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index 50694faf486..2f030b19c5a 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -1,24 +1,29 @@ # newer versions go on top +- version: "2.7.0" + changes: + - description: Improve handling of pagination end. + type: enhancement + link: https://github.com/elastic/integrations/pull/13965 - version: "2.6.1" changes: - description: Fix audit log URL for enterprises. type: bugfix - link: http://github.com/elastic/integrations/pull/13183 + link: https://github.com/elastic/integrations/pull/13183 - version: "2.6.0" changes: - description: Update audit ingest pipeline to cover all fields. type: enhancement - link: http://github.com/elastic/integrations/pull/13092 + link: https://github.com/elastic/integrations/pull/13092 - version: "2.5.0" changes: - description: Add Azure Eventhub support for audit logs. type: enhancement - link: http://github.com/elastic/integrations/pull/13038 + link: https://github.com/elastic/integrations/pull/13038 - version: "2.4.1" changes: - description: Add missing ECS field in latest_code_scanning transform. type: bugfix - link: http://github.com/elastic/integrations/pull/12841 + link: https://github.com/elastic/integrations/pull/12841 - version: "2.4.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/github/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/github/data_stream/audit/agent/stream/httpjson.yml.hbs index 37b9c68dfcf..67c67e28727 100644 --- a/packages/github/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/github/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -51,6 +51,7 @@ response.pagination: target: url.value value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' fail_on_template_error: true + do_not_log_failure: true cursor: last_timestamp: diff --git a/packages/github/data_stream/code_scanning/agent/stream/httpjson.yml.hbs b/packages/github/data_stream/code_scanning/agent/stream/httpjson.yml.hbs index 4df1bb39a42..a064cbf6c9e 100644 --- a/packages/github/data_stream/code_scanning/agent/stream/httpjson.yml.hbs +++ b/packages/github/data_stream/code_scanning/agent/stream/httpjson.yml.hbs @@ -34,6 +34,7 @@ response.pagination: target: url.value value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' fail_on_template_error: true + do_not_log_failure: true {{#if tags.length}} tags: diff --git a/packages/github/data_stream/dependabot/agent/stream/httpjson.yml.hbs b/packages/github/data_stream/dependabot/agent/stream/httpjson.yml.hbs index d6ae2bbba59..7aca63fe7e4 100644 --- a/packages/github/data_stream/dependabot/agent/stream/httpjson.yml.hbs +++ b/packages/github/data_stream/dependabot/agent/stream/httpjson.yml.hbs @@ -43,11 +43,13 @@ response.pagination: target: body.query value: "query fetchRepoAlerts ($org: String!, $repo:String!, $cursor:String!) { repository(owner: $org, name: $repo) { vulnerabilityAlerts(first: 100, after: $cursor) { nodes { createdAt dependabotUpdate { error { body errorType title } pullRequest { createdAt closed closedAt merged mergedAt number url title } } dependencyScope dismissReason dismissedAt dismisser { login url } fixedAt number repository { description isInOrganization isPrivate name owner { login url } url } securityAdvisory { classification cvss { score vectorString } cwes(first:100) { nodes { cweId description name } } description ghsaId identifiers { type value } origin permalink references { url } publishedAt severity summary updatedAt withdrawnAt } securityVulnerability { firstPatchedVersion { identifier } package { ecosystem name } severity updatedAt vulnerableVersionRange } state vulnerableManifestPath vulnerableManifestFilename vulnerableRequirements } pageInfo { hasNextPage endCursor } } } }" fail_on_template_error: true + do_not_log_failure: true - set: target: body.variables value: "{\"org\": \"{{owner}}\", \"repo\": \"{{repo}}\", \"cursor\": \"[[ .last_response.body.data.repository.vulnerabilityAlerts.pageInfo.endCursor ]]\"}" fail_on_template_error: true + do_not_log_failure: true {{else}} request.body: query: @@ -71,11 +73,13 @@ response.pagination: target: body.query value: "query fetchRepoAlerts ($org: String!, $cursor:String!) { organization(login: $org) { repositories(first: 10, after: $cursor) { nodes { vulnerabilityAlerts(first: 100) { nodes { createdAt dependabotUpdate { error { body errorType title } pullRequest { createdAt closed closedAt merged mergedAt number url title } } dependencyScope dismissReason dismissedAt dismisser { login url } fixedAt number repository { description isInOrganization isPrivate name owner { login url } url } securityAdvisory { classification cvss { score vectorString } cwes(first:2) { nodes { cweId description name } } description ghsaId identifiers { type value } origin permalink references { url } publishedAt severity summary updatedAt withdrawnAt } securityVulnerability { firstPatchedVersion { identifier } package { ecosystem name } severity updatedAt vulnerableVersionRange } state vulnerableManifestPath vulnerableManifestFilename vulnerableRequirements } pageInfo { hasNextPage endCursor } } } pageInfo { hasNextPage endCursor } } } }" fail_on_template_error: true + do_not_log_failure: true - set: target: body.variables value: "{\"org\": \"{{owner}}\", \"cursor\": \"[[ .last_response.body.data.organization.repositories.pageInfo.endCursor ]]\"}" fail_on_template_error: true + do_not_log_failure: true {{/if}} diff --git a/packages/github/data_stream/issues/agent/stream/httpjson.yml.hbs b/packages/github/data_stream/issues/agent/stream/httpjson.yml.hbs index af77466dca7..fc42d589305 100644 --- a/packages/github/data_stream/issues/agent/stream/httpjson.yml.hbs +++ b/packages/github/data_stream/issues/agent/stream/httpjson.yml.hbs @@ -49,6 +49,7 @@ response.pagination: target: url.value value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' fail_on_template_error: true + do_not_log_failure: true {{#if tags.length}} tags: diff --git a/packages/github/data_stream/secret_scanning/agent/stream/httpjson.yml.hbs b/packages/github/data_stream/secret_scanning/agent/stream/httpjson.yml.hbs index 22481bca763..1738eb66609 100644 --- a/packages/github/data_stream/secret_scanning/agent/stream/httpjson.yml.hbs +++ b/packages/github/data_stream/secret_scanning/agent/stream/httpjson.yml.hbs @@ -37,6 +37,7 @@ response.pagination: target: url.value value: '[[ getRFC5988Link "next" .last_response.header.Link ]]' fail_on_template_error: true + do_not_log_failure: true {{#if tags.length}} tags: diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml index e450aa5e377..bb9b313c41f 100644 --- a/packages/github/manifest.yml +++ b/packages/github/manifest.yml @@ -1,13 +1,13 @@ name: github title: GitHub -version: "2.6.1" +version: "2.7.0" description: Collect logs from GitHub with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, "productivity_security"] conditions: kibana: - version: "^8.13.0 || ^9.0.0" + version: "^8.15.0 || ^9.0.0" icons: - src: /img/github.svg title: GitHub