From 2356d100336b4644cdd547f13ed123f31bb10d3a Mon Sep 17 00:00:00 2001 From: Linu-Elias Date: Sat, 21 Sep 2024 12:27:07 +0530 Subject: [PATCH 1/2] x-pack/metricbeat/module/gcp: Override GCP API endpoint in metric client (#40918) * override endpoint to etric client * config and changelog * doc * resolved comments * lint fix (cherry picked from commit 7720c900f8933cbfdf15321479ff71220ac364cd) --- CHANGELOG.next.asciidoc | 2 ++ metricbeat/docs/modules/gcp.asciidoc | 3 +++ x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/gcp/_meta/config.yml | 1 + x-pack/metricbeat/module/gcp/_meta/docs.asciidoc | 2 ++ x-pack/metricbeat/module/gcp/metrics/metricset.go | 6 ++++++ x-pack/metricbeat/modules.d/gcp.yml.disabled | 1 + 7 files changed, 16 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8f3defa5ac38..b0aaedd92d78 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -62,6 +62,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Mark system process metricsets as running if metrics are partially available {pull}40565[40565] - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571] - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683] +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] +- Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918] *Osquerybeat* diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 5f81455412ac..025b70a94040 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -52,6 +52,8 @@ and metadata information from metricsets and fetch metrics only. At the moment, * *period*: A single time duration specified for this module collection frequency. +* *endpoint*: A custom endpoint to use for the GCP API calls. If not specified, the default endpoint will be used. + [float] == Example configuration * `compute` metricset is enabled to collect metrics from `us-central1-a` zone @@ -334,6 +336,7 @@ metricbeat.modules: period: 24h project_id: "your project id" credentials_file_path: "your JSON credentials file path" + endpoint: http://your-endpoint dataset_id: "dataset id" table_pattern: "table pattern" ---- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index c1821be998b2..77b1a22287d2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -641,6 +641,7 @@ metricbeat.modules: period: 24h project_id: "your project id" credentials_file_path: "your JSON credentials file path" + endpoint: http://your-endpoint dataset_id: "dataset id" table_pattern: "table pattern" diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index f4e245080aeb..ad0c7da852fa 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -67,5 +67,6 @@ period: 24h project_id: "your project id" credentials_file_path: "your JSON credentials file path" + endpoint: http://your-endpoint dataset_id: "dataset id" table_pattern: "table pattern" diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index ae9431a23fd6..27a326fe78d3 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -40,6 +40,8 @@ and metadata information from metricsets and fetch metrics only. At the moment, * *period*: A single time duration specified for this module collection frequency. +* *endpoint*: A custom endpoint to use for the GCP API calls. If not specified, the default endpoint will be used. + [float] == Example configuration * `compute` metricset is enabled to collect metrics from `us-central1-a` zone diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index d56abf27acbf..274a6153b54c 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -105,6 +105,7 @@ type config struct { ExcludeLabels bool `config:"exclude_labels"` CredentialsFilePath string `config:"credentials_file_path"` CredentialsJSON string `config:"credentials_json"` + Endpoint string `config:"endpoint"` opt []option.ClientOption period *durationpb.Duration @@ -139,6 +140,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return m, fmt.Errorf("no credentials_file_path or credentials_json specified") } + if m.config.Endpoint != "" { + m.Logger().Warnf("You are using a custom endpoint '%s' for the GCP API calls.", m.config.Endpoint) + m.config.opt = append(m.config.opt, option.WithEndpoint(m.config.Endpoint)) + } + m.config.period = &durationpb.Duration{ Seconds: int64(m.Module().Config().Period.Seconds()), } diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index f79e1607a453..c8cddd198ce3 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -70,5 +70,6 @@ period: 24h project_id: "your project id" credentials_file_path: "your JSON credentials file path" + endpoint: http://your-endpoint dataset_id: "dataset id" table_pattern: "table pattern" From f19f982346428c60bcee2f2408685dc2e4a4ce1b Mon Sep 17 00:00:00 2001 From: Linu-Elias Date: Mon, 23 Sep 2024 10:37:59 +0530 Subject: [PATCH 2/2] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b0aaedd92d78..62bf4a822e76 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -62,7 +62,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Mark system process metricsets as running if metrics are partially available {pull}40565[40565] - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571] - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683] -- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] - Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918] *Osquerybeat*