From 3dbb043e743fad09cac0dc474b404bd0de9082e5 Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Fri, 26 Jul 2024 12:34:55 +0200 Subject: [PATCH 1/7] [kibana] support api_key --- metricbeat/docs/modules/kibana.asciidoc | 3 +++ metricbeat/metricbeat.reference.yml | 3 +++ metricbeat/module/kibana/_meta/config-xpack.yml | 1 + metricbeat/module/kibana/_meta/config.reference.yml | 3 +++ metricbeat/module/kibana/_meta/config.yml | 1 + .../module/kibana/cluster_actions/cluster_actions.go | 2 +- .../module/kibana/cluster_rules/cluster_rules.go | 2 +- metricbeat/module/kibana/config.go | 2 ++ metricbeat/module/kibana/kibana.go | 7 ++++--- metricbeat/module/kibana/metricset.go | 11 +++++++++++ metricbeat/module/kibana/node_actions/node_actions.go | 2 +- metricbeat/module/kibana/node_rules/node_rules.go | 2 +- metricbeat/module/kibana/settings/settings.go | 8 +++++++- metricbeat/module/kibana/stats/stats.go | 2 +- metricbeat/module/kibana/status/status.go | 2 ++ metricbeat/modules.d/kibana-xpack.yml.disabled | 1 + metricbeat/modules.d/kibana.yml.disabled | 1 + x-pack/metricbeat/metricbeat.reference.yml | 3 +++ 18 files changed, 47 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc index 6757d8db80e7..70c1054f463f 100644 --- a/metricbeat/docs/modules/kibana.asciidoc +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -51,6 +51,9 @@ metricbeat.modules: hosts: ["localhost:5601"] basepath: "" enabled: true + #username: "user" + #password: "secret" + #api_key: "foo:bar" # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 985e4795910c..cb66c814f705 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -493,6 +493,9 @@ metricbeat.modules: hosts: ["localhost:5601"] basepath: "" enabled: true + #username: "user" + #password: "secret" + #api_key: "foo:bar" # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. diff --git a/metricbeat/module/kibana/_meta/config-xpack.yml b/metricbeat/module/kibana/_meta/config-xpack.yml index 73dffedb072f..d45d2895b034 100644 --- a/metricbeat/module/kibana/_meta/config-xpack.yml +++ b/metricbeat/module/kibana/_meta/config-xpack.yml @@ -5,3 +5,4 @@ #basepath: "" #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/module/kibana/_meta/config.reference.yml b/metricbeat/module/kibana/_meta/config.reference.yml index 466dd7899dd1..3dbb32852379 100644 --- a/metricbeat/module/kibana/_meta/config.reference.yml +++ b/metricbeat/module/kibana/_meta/config.reference.yml @@ -4,6 +4,9 @@ hosts: ["localhost:5601"] basepath: "" enabled: true + #username: "user" + #password: "secret" + #api_key: "foo:bar" # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml index f997985d3c2c..3b4317c527f2 100644 --- a/metricbeat/module/kibana/_meta/config.yml +++ b/metricbeat/module/kibana/_meta/config.yml @@ -6,3 +6,4 @@ #basepath: "" #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/module/kibana/cluster_actions/cluster_actions.go b/metricbeat/module/kibana/cluster_actions/cluster_actions.go index cb50ce6c21a6..b4e1e6e4fe70 100644 --- a/metricbeat/module/kibana/cluster_actions/cluster_actions.go +++ b/metricbeat/module/kibana/cluster_actions/cluster_actions.go @@ -88,7 +88,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { } func (m *MetricSet) validate() (err error, versionSupported bool) { - kibanaVersion, err := kibana.GetVersion(m.actionsHTTP, kibana.ClusterActionsPath) + kibanaVersion, err := kibana.GetVersion(m.actionsHTTP, kibana.ClusterActionsPath, m.ApiKey) if err != nil { return err, false } diff --git a/metricbeat/module/kibana/cluster_rules/cluster_rules.go b/metricbeat/module/kibana/cluster_rules/cluster_rules.go index 6131b5cdc5e0..bbb9b182cc46 100644 --- a/metricbeat/module/kibana/cluster_rules/cluster_rules.go +++ b/metricbeat/module/kibana/cluster_rules/cluster_rules.go @@ -88,7 +88,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { } func (m *MetricSet) validate() (err error, versionSupported bool) { - kibanaVersion, err := kibana.GetVersion(m.rulesHTTP, kibana.ClusterRulesPath) + kibanaVersion, err := kibana.GetVersion(m.rulesHTTP, kibana.ClusterRulesPath, m.ApiKey) if err != nil { return err, false } diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go index 748ba6bb5e31..c1653ec7dc03 100644 --- a/metricbeat/module/kibana/config.go +++ b/metricbeat/module/kibana/config.go @@ -20,11 +20,13 @@ package kibana // Config defines the structure for the Kibana module configuration options type Config struct { XPackEnabled bool `config:"xpack.enabled"` + ApiKey string `config:"api_key"` } // DefaultConfig returns the default configuration for the Kibana module func DefaultConfig() Config { return Config{ XPackEnabled: false, + ApiKey: "", } } diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 9a3fa60b12e8..7ce86e79e8ed 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -96,8 +96,8 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { } // GetVersion returns the version of the Kibana instance -func GetVersion(http *helper.HTTP, currentPath string) (*version.V, error) { - content, err := fetchPath(http, currentPath, StatusPath) +func GetVersion(http *helper.HTTP, currentPath string, apiKey string) (*version.V, error) { + content, err := fetchPath(http, currentPath, StatusPath, apiKey) if err != nil { return nil, err } @@ -144,7 +144,7 @@ func IsUsageExcludable(currentKibanaVersion *version.V) bool { v7_0_1.LessThanOrEqual(false, currentKibanaVersion) } -func fetchPath(http *helper.HTTP, currentPath, newPath string) ([]byte, error) { +func fetchPath(http *helper.HTTP, currentPath, newPath string, apiKey string) ([]byte, error) { currentURI := http.GetURI() defer http.SetURI(currentURI) // Reset after this request @@ -159,5 +159,6 @@ func fetchPath(http *helper.HTTP, currentPath, newPath string) ([]byte, error) { // Http helper includes the HostData with username and password http.SetURI(u.String()) + http.SetHeader("Authorization", "ApiKey " + apiKey) return http.FetchContent() } diff --git a/metricbeat/module/kibana/metricset.go b/metricbeat/module/kibana/metricset.go index 1a68669fe58e..18cf5d3b7268 100644 --- a/metricbeat/module/kibana/metricset.go +++ b/metricbeat/module/kibana/metricset.go @@ -18,13 +18,16 @@ package kibana import ( + "encoding/base64" "github.com/elastic/beats/v7/metricbeat/mb" + "fmt" ) // MetricSet can be used to build other metricsets within the Kibana module. type MetricSet struct { mb.BaseMetricSet XPackEnabled bool + ApiKey string } // NewMetricSet creates a metricset that can be used to build other metricsets @@ -35,8 +38,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } + if config.ApiKey != "" { + hostData := base.HostData() + if hostData.User != "" || hostData.Password != "" { + return nil, fmt.Errorf("cannot set both api_key and username/password") + } + } + return &MetricSet{ base, config.XPackEnabled, + base64.StdEncoding.EncodeToString([]byte(config.ApiKey)), }, nil } diff --git a/metricbeat/module/kibana/node_actions/node_actions.go b/metricbeat/module/kibana/node_actions/node_actions.go index f6f742ccffa1..83d8b97d4df4 100644 --- a/metricbeat/module/kibana/node_actions/node_actions.go +++ b/metricbeat/module/kibana/node_actions/node_actions.go @@ -88,7 +88,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { } func (m *MetricSet) validate() (err error, versionSupported bool) { - kibanaVersion, err := kibana.GetVersion(m.actionsHTTP, kibana.NodeActionsPath) + kibanaVersion, err := kibana.GetVersion(m.actionsHTTP, kibana.NodeActionsPath, m.ApiKey) if err != nil { return err, false } diff --git a/metricbeat/module/kibana/node_rules/node_rules.go b/metricbeat/module/kibana/node_rules/node_rules.go index 5af7234c54bc..34fe0f912682 100644 --- a/metricbeat/module/kibana/node_rules/node_rules.go +++ b/metricbeat/module/kibana/node_rules/node_rules.go @@ -88,7 +88,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { } func (m *MetricSet) validate() (err error, versionSupported bool) { - kibanaVersion, err := kibana.GetVersion(m.rulesHTTP, kibana.NodeRulesPath) + kibanaVersion, err := kibana.GetVersion(m.rulesHTTP, kibana.NodeRulesPath, m.ApiKey) if err != nil { return err, false } diff --git a/metricbeat/module/kibana/settings/settings.go b/metricbeat/module/kibana/settings/settings.go index b2468bfa461c..e85abb4b3128 100644 --- a/metricbeat/module/kibana/settings/settings.go +++ b/metricbeat/module/kibana/settings/settings.go @@ -45,13 +45,19 @@ var ( // MetricSet type defines all fields of the MetricSet type MetricSet struct { + *kibana.MetricSet mb.BaseMetricSet settingsHTTP *helper.HTTP } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := kibana.NewMetricSet(base) + if err != nil { + return nil, err + } return &MetricSet{ + MetricSet: ms, BaseMetricSet: base, }, nil } @@ -80,7 +86,7 @@ func (m *MetricSet) init() (err error) { httpHelper.SetHeaderDefault(productorigin.Header, productorigin.Beats) - kibanaVersion, err := kibana.GetVersion(httpHelper, kibana.SettingsPath) + kibanaVersion, err := kibana.GetVersion(httpHelper, kibana.SettingsPath, m.ApiKey) if err != nil { return err } diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 5b96525b0e56..6659bf6eddff 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -93,7 +93,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { } func (m *MetricSet) init() (err error, versionSupported bool) { - kibanaVersion, err := kibana.GetVersion(m.statsHTTP, kibana.StatsPath) + kibanaVersion, err := kibana.GetVersion(m.statsHTTP, kibana.StatsPath, m.ApiKey) if err != nil { return err, false } diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index 7715ca48cb91..ef09e4bcba23 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -72,6 +72,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { + apiKey := m.ApiKey + m.http.SetHeader("Authorization", "ApiKey " + apiKey) content, err := m.http.FetchContent() if err != nil { return err diff --git a/metricbeat/modules.d/kibana-xpack.yml.disabled b/metricbeat/modules.d/kibana-xpack.yml.disabled index 91471a7c212b..7bc747f0fe97 100644 --- a/metricbeat/modules.d/kibana-xpack.yml.disabled +++ b/metricbeat/modules.d/kibana-xpack.yml.disabled @@ -8,3 +8,4 @@ #basepath: "" #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/metricbeat/modules.d/kibana.yml.disabled b/metricbeat/modules.d/kibana.yml.disabled index 27ca4b1a05fe..138d3bd6e607 100644 --- a/metricbeat/modules.d/kibana.yml.disabled +++ b/metricbeat/modules.d/kibana.yml.disabled @@ -9,3 +9,4 @@ #basepath: "" #username: "user" #password: "secret" + #api_key: "foo:bar" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1d2957306db9..3fd44b70fc24 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -890,6 +890,9 @@ metricbeat.modules: hosts: ["localhost:5601"] basepath: "" enabled: true + #username: "elastic" + #password: "changeme" + #api_key: "foo:bar" # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. From 16c3798ca962e3485d82fc9f47c7b01325b9cf40 Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Wed, 11 Sep 2024 14:30:38 +0200 Subject: [PATCH 2/7] changes username and password to generic values --- x-pack/metricbeat/metricbeat.reference.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 68b5b819b066..eea5852e8644 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -890,8 +890,8 @@ metricbeat.modules: hosts: ["localhost:5601"] basepath: "" enabled: true - #username: "elastic" - #password: "changeme" + #username: "user" + #password: "secret" #api_key: "foo:bar" # Set to true to send data collected by module to X-Pack From a1059350e2f4a8564d147f80b989e8010ee25456 Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Wed, 11 Sep 2024 16:14:41 +0200 Subject: [PATCH 3/7] make check formatting changes --- metricbeat/module/kibana/config.go | 4 ++-- metricbeat/module/kibana/kibana.go | 2 +- metricbeat/module/kibana/metricset.go | 3 ++- metricbeat/module/kibana/settings/settings.go | 2 +- metricbeat/module/kibana/status/status.go | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go index c1653ec7dc03..e559d946856e 100644 --- a/metricbeat/module/kibana/config.go +++ b/metricbeat/module/kibana/config.go @@ -19,7 +19,7 @@ package kibana // Config defines the structure for the Kibana module configuration options type Config struct { - XPackEnabled bool `config:"xpack.enabled"` + XPackEnabled bool `config:"xpack.enabled"` ApiKey string `config:"api_key"` } @@ -27,6 +27,6 @@ type Config struct { func DefaultConfig() Config { return Config{ XPackEnabled: false, - ApiKey: "", + ApiKey: "", } } diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 7ce86e79e8ed..3e36c0feaafc 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -159,6 +159,6 @@ func fetchPath(http *helper.HTTP, currentPath, newPath string, apiKey string) ([ // Http helper includes the HostData with username and password http.SetURI(u.String()) - http.SetHeader("Authorization", "ApiKey " + apiKey) + http.SetHeader("Authorization", "ApiKey "+apiKey) return http.FetchContent() } diff --git a/metricbeat/module/kibana/metricset.go b/metricbeat/module/kibana/metricset.go index 18cf5d3b7268..f547373cb6b8 100644 --- a/metricbeat/module/kibana/metricset.go +++ b/metricbeat/module/kibana/metricset.go @@ -19,8 +19,9 @@ package kibana import ( "encoding/base64" - "github.com/elastic/beats/v7/metricbeat/mb" "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" ) // MetricSet can be used to build other metricsets within the Kibana module. diff --git a/metricbeat/module/kibana/settings/settings.go b/metricbeat/module/kibana/settings/settings.go index e85abb4b3128..2bc9c9da9b54 100644 --- a/metricbeat/module/kibana/settings/settings.go +++ b/metricbeat/module/kibana/settings/settings.go @@ -57,7 +57,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } return &MetricSet{ - MetricSet: ms, + MetricSet: ms, BaseMetricSet: base, }, nil } diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index ef09e4bcba23..149a8f09ac12 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -73,7 +73,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { apiKey := m.ApiKey - m.http.SetHeader("Authorization", "ApiKey " + apiKey) + m.http.SetHeader("Authorization", "ApiKey "+apiKey) content, err := m.http.FetchContent() if err != nil { return err From 0730c5fd5b60a88bffd18fbabfef20d000e973dc Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Wed, 11 Sep 2024 16:33:36 +0200 Subject: [PATCH 4/7] removes skip-go-installation option --- .github/workflows/golangci-lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 94059ab8d963..ad833a3d99c3 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -53,8 +53,5 @@ jobs: # into fixing all linting issues in the whole file instead args: --timeout=30m --whole-files - # Optional: if set to true then the action will use pre-installed Go. - skip-go-installation: true - # Optional: show only new issues if it's a pull request. The default value is `false`. only-new-issues: true From 2a8dcbd20049958efcebbe97bc3e2f871395ec49 Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Wed, 11 Sep 2024 16:54:17 +0200 Subject: [PATCH 5/7] fixes: naked return in func --- metricbeat/module/kibana/settings/settings.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/settings/settings.go b/metricbeat/module/kibana/settings/settings.go index 2bc9c9da9b54..2d8b9f943ea7 100644 --- a/metricbeat/module/kibana/settings/settings.go +++ b/metricbeat/module/kibana/settings/settings.go @@ -67,12 +67,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { if err = m.init(); err != nil { - return + return err } content, err := m.settingsHTTP.FetchContent() if err != nil { - return + return err } return eventMapping(r, content) @@ -99,5 +99,5 @@ func (m *MetricSet) init() (err error) { m.settingsHTTP, err = helper.NewHTTP(m.BaseMetricSet) - return + return err } From 6fd0646b1d09e59087e3f1d4ebed3279e688d4ee Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Mon, 16 Sep 2024 14:38:47 +0200 Subject: [PATCH 6/7] adds skip-go-installation back --- .github/workflows/golangci-lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ad833a3d99c3..94059ab8d963 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -53,5 +53,8 @@ jobs: # into fixing all linting issues in the whole file instead args: --timeout=30m --whole-files + # Optional: if set to true then the action will use pre-installed Go. + skip-go-installation: true + # Optional: show only new issues if it's a pull request. The default value is `false`. only-new-issues: true From 8a4b6ee8f2fb9feec2b01984cd8617395e5fb931 Mon Sep 17 00:00:00 2001 From: sachin-frayne Date: Wed, 18 Sep 2024 11:02:45 +0200 Subject: [PATCH 7/7] only sets header if apikey is not empty --- metricbeat/module/kibana/kibana.go | 4 +++- metricbeat/module/kibana/status/status.go | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 3e36c0feaafc..499f5c5ede57 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -159,6 +159,8 @@ func fetchPath(http *helper.HTTP, currentPath, newPath string, apiKey string) ([ // Http helper includes the HostData with username and password http.SetURI(u.String()) - http.SetHeader("Authorization", "ApiKey "+apiKey) + if apiKey != "" { + http.SetHeader("Authorization", "ApiKey "+apiKey) + } return http.FetchContent() } diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index 149a8f09ac12..62a1a0b0e4d6 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -72,8 +72,9 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { - apiKey := m.ApiKey - m.http.SetHeader("Authorization", "ApiKey "+apiKey) + if m.ApiKey != "" { + m.http.SetHeader("Authorization", "ApiKey "+m.ApiKey) + } content, err := m.http.FetchContent() if err != nil { return err