From 458f523ea553ad11a206e1fce2e655416119348b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 12 Nov 2023 17:30:25 +0100 Subject: [PATCH 01/53] adds pgbouncer module --- metricbeat/include/list_common.go | 2 + .../pgbouncer/_meta/config.reference.yml | 0 metricbeat/module/pgbouncer/_meta/config.yml | 15 +++ metricbeat/module/pgbouncer/_meta/fields.yml | 12 +++ metricbeat/module/pgbouncer/doc.go | 21 ++++ .../module/pgbouncer/docker-compose.yml | 0 metricbeat/module/pgbouncer/metricset.go | 74 +++++++++++++ metricbeat/module/pgbouncer/pgbouncer.go | 102 ++++++++++++++++++ .../module/pgbouncer/stats/_meta/fields.yml | 68 ++++++++++++ metricbeat/module/pgbouncer/stats/data.go | 42 ++++++++ metricbeat/module/pgbouncer/stats/stats.go | 42 ++++++++ 11 files changed, 378 insertions(+) create mode 100644 metricbeat/module/pgbouncer/_meta/config.reference.yml create mode 100644 metricbeat/module/pgbouncer/_meta/config.yml create mode 100644 metricbeat/module/pgbouncer/_meta/fields.yml create mode 100644 metricbeat/module/pgbouncer/doc.go create mode 100644 metricbeat/module/pgbouncer/docker-compose.yml create mode 100644 metricbeat/module/pgbouncer/metricset.go create mode 100644 metricbeat/module/pgbouncer/pgbouncer.go create mode 100644 metricbeat/module/pgbouncer/stats/_meta/fields.yml create mode 100644 metricbeat/module/pgbouncer/stats/data.go create mode 100644 metricbeat/module/pgbouncer/stats/stats.go diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 434d2d7fc72c..c63d6ceac672 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -191,4 +191,6 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/connection" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/mntr" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/server" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/stats" ) diff --git a/metricbeat/module/pgbouncer/_meta/config.reference.yml b/metricbeat/module/pgbouncer/_meta/config.reference.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml new file mode 100644 index 000000000000..ee23ba65011f --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -0,0 +1,15 @@ +# Module: pgbouncer +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html + +metricbeat.modules: +- module: pgbouncer + metricsets: + - stats + period: 10s + hosts: ["pgbouncer://localhost:5432"] + username: test + password: password +output.elasticsearch: + hosts: ["http://172.21.186.195:9200"] + username: elastic + password: xxxx \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/fields.yml b/metricbeat/module/pgbouncer/_meta/fields.yml new file mode 100644 index 000000000000..9d2e749a695e --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/fields.yml @@ -0,0 +1,12 @@ +- key: pgbouncer + title: "PGBouncer" + description: > + Metrics collected from pgbouncer. + short_config: false + release: ga + fields: + - name: pgbouncer + type: group + description: > + PGBouncer metrics. + fields: \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/doc.go b/metricbeat/module/pgbouncer/doc.go new file mode 100644 index 000000000000..d87b920f355d --- /dev/null +++ b/metricbeat/module/pgbouncer/doc.go @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package postgresql is a Metricbeat module that contains MetricSets. +*/ +package pgbouncer \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/metricbeat/module/pgbouncer/metricset.go b/metricbeat/module/pgbouncer/metricset.go new file mode 100644 index 000000000000..2083111e7d7e --- /dev/null +++ b/metricbeat/module/pgbouncer/metricset.go @@ -0,0 +1,74 @@ +package pgbouncer +import ( + "context" + "database/sql" + "fmt" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/logp" + // Register pgbouncer database/sql driver + _ "github.com/lib/pq" +) +type MetricSet struct { + mb.BaseMetricSet + db *sql.DB +} +// NewMetricSet creates a pgbouncer metricset with a pool of connections +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + return &MetricSet{BaseMetricSet: base}, nil +} +// DB creates a database connection, it must be freed after use with `Close()` +func (ms *MetricSet) DB(ctx context.Context) (*sql.Conn, error) { + if ms.db == nil { + db, err := sql.Open("postgres", ms.HostData().URI) + if err != nil { + return nil, fmt.Errorf("failed to open connection: %w", err) + } + ms.db = db + } + return ms.db.Conn(ctx) +} +// QueryStats makes the database call for a given metric +func (ms *MetricSet) QueryStats(ctx context.Context, query string) ([]map[string]interface{}, error) { + db, err := ms.DB(ctx) + if err != nil { + return nil, fmt.Errorf("failed to obtain a connection with the database: %w", err) + } + defer db.Close() + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query database: %w", err) + } + columns, err := rows.Columns() + if err != nil { + return nil, fmt.Errorf("scanning columns: %w", err) + } + vals := make([][]byte, len(columns)) + valPointers := make([]interface{}, len(columns)) + for i := range vals { + valPointers[i] = &vals[i] + } + results := []map[string]interface{}{} + for rows.Next() { + err = rows.Scan(valPointers...) + if err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + result := map[string]interface{}{} + for i, col := range columns { + result[col] = string(vals[i]) + } + logp.Debug("postgresql", "Result: %v", result) + results = append(results, result) + } + return results, nil +} +// Close closes the metricset and its connections +func (ms *MetricSet) Close() error { + if ms.db == nil { + return nil + } + if err := ms.db.Close(); err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + return nil +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pgbouncer.go b/metricbeat/module/pgbouncer/pgbouncer.go new file mode 100644 index 000000000000..3b7cf25cc8ce --- /dev/null +++ b/metricbeat/module/pgbouncer/pgbouncer.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package pgbouncer is Metricbeat module for PostgreSQL server. +*/ +package pgbouncer + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/lib/pq" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +func init() { + // Register the ModuleFactory function for the "pgbouncer" module. + if err := mb.Registry.AddModule("pgbouncer", NewModule); err != nil { + panic(err) + } +} + +// NewModule returns a new instance of the module +func NewModule(base mb.BaseModule) (mb.Module, error) { + // Validate that at least one host has been specified. + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + }{} + if err := base.UnpackConfig(&config); err != nil { + return nil, err + } + + return &base, nil +} + +// ParseURL is the postgres host parser +func ParseURL(mod mb.Module, rawURL string) (mb.HostData, error) { + c := struct { + Username string `config:"username"` + Password string `config:"password"` + }{} + if err := mod.UnpackConfig(&c); err != nil { + return mb.HostData{}, err + } + + if parts := strings.SplitN(rawURL, "://", 2); len(parts) != 2 { + // Add scheme. + rawURL = fmt.Sprintf("postgres://%s", rawURL) + } + + u, err := url.Parse(rawURL) + if err != nil { + return mb.HostData{}, fmt.Errorf("error parsing URL: %v", err) + } + + parse.SetURLUser(u, c.Username, c.Password) + + if timeout := mod.Config().Timeout; timeout > 0 { + q := u.Query() + q.Set("connect_timeout", strconv.Itoa(int(timeout.Seconds()))) + u.RawQuery = q.Encode() + } + + // https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING + connString, err := pq.ParseURL(u.String()) + if err != nil { + return mb.HostData{}, err + } + + h := parse.NewHostDataFromURL(u) + + // Store the connection string instead of URL to avoid the cost of sql.Open + // parsing the URL on each call. + h.URI = connString + + // Postgres URLs can use a host query param to specify the host. This is + // used for unix domain sockets (postgres:///dbname?host=/var/lib/postgres). + if host := u.Query().Get("host"); u.Host == "" && host != "" { + h.Host = host + } + + return h, nil +} diff --git a/metricbeat/module/pgbouncer/stats/_meta/fields.yml b/metricbeat/module/pgbouncer/stats/_meta/fields.yml new file mode 100644 index 000000000000..8836ce25ddc2 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/fields.yml @@ -0,0 +1,68 @@ +- name: stats + type: group + description: > + One document per server process, showing information related to the current + activity of that process, such as state and current query. Collected by + querying stats. + release: ga + fields: + - name: database + type: keyword + description: > + Name of the database this backend is connected to. + - name: total_query_count + type: long + description: > + OID of the database this backend is connected to. + - name: total_received + type: long + description: > + OID of the database this backend is connected to. + - name: total_sent + type: long + description: > + OID of the database this backend is connected to. + - name: total_xact_time + type: long + description: > + OID of the database this backend is connected to. + - name: total_query_time + type: long + description: > + OID of the database this backend is connected to. + - name: total_wait_time + type: long + description: > + OID of the database this backend is connected to. + - name: total_xact_count + type: long + description: > + OID of the database this backend is connected to. + - name: avg_xact_count + type: long + description: > + OID of the database this backend is connected to. + - name: avg_query_count + type: long + description: > + OID of the database this backend is connected to. + - name: avg_recv + type: long + description: > + OID of the database this backend is connected to. + - name: avg_sent + type: long + description: > + OID of the database this backend is connected to. + - name: avg_xact_time + type: long + description: > + OID of the database this backend is connected to. + - name: avg_query_time + type: long + description: > + OID of the database this backend is connected to. + - name: avg_wait_time + type: long + description: > + OID of the database this backend is connected to. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go new file mode 100644 index 000000000000..b03f76e3f094 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +// Based on pgbouncer show stats; +var schema = s.Schema{ + "database": c.Str("database"), + "total_query_count": c.Int("total_query_count"), + "total_received": c.Int("total_received"), + "total_sent": c.Int("total_sent"), + "total_xact_time": c.Int("total_xact_time"), + "total_query_time": c.Int("total_query_time"), + "total_wait_time": c.Int("total_wait_time"), + "total_xact_count": c.Int("total_xact_count"), + "avg_xact_count": c.Int("avg_xact_count"), + "avg_query_count": c.Int("avg_query_count"), + "avg_recv": c.Int("avg_recv"), + "avg_sent": c.Int("avg_sent"), + "avg_xact_time": c.Int("avg_xact_time"), + "avg_query_time": c.Int("avg_query_time"), + "avg_wait_time": c.Int("avg_wait_time"), +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go new file mode 100644 index 000000000000..3bbc83fdef8b --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -0,0 +1,42 @@ +package stats +import ( + "context" + "fmt" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" +) +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "stats", New, + mb.WithHostParser(postgresql.ParseURL), + mb.DefaultMetricSet(), + ) +} +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *postgresql.MetricSet +} +// New create a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := postgresql.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + ctx := context.Background() + results, err := m.QueryStats(ctx, "SHOW STATS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + if len(results) == 0 { + return fmt.Errorf("No results from the stats query") + } + data, _ := schema.Apply(results[0]) + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + return nil +} \ No newline at end of file From c366ea02ab9c37935b4d50752d87b15a2455db5e Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:29 +0200 Subject: [PATCH 02/53] adds pgbouncer module --- metricbeat/include/list_common.go | 4 +- metricbeat/metricbeat.reference.yml | 2 + metricbeat/module/pgbouncer/_meta/Dockerfile | 20 ++++++++ metricbeat/module/pgbouncer/_meta/config.yml | 6 ++- metricbeat/module/pgbouncer/doc.go | 2 +- .../module/pgbouncer/docker-compose.yml | 48 +++++++++++++++++++ metricbeat/module/pgbouncer/fields.go | 36 ++++++++++++++ metricbeat/modules.d/pgbouncer.yml.disabled | 20 ++++++++ 8 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 metricbeat/module/pgbouncer/_meta/Dockerfile create mode 100644 metricbeat/module/pgbouncer/fields.go create mode 100644 metricbeat/modules.d/pgbouncer.yml.disabled diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index c63d6ceac672..7b698406e07b 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -135,6 +135,8 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/nginx/stubstatus" _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics" _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics/collector" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/stats" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm/pool" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm/process" @@ -191,6 +193,4 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/connection" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/mntr" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/server" - _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" - _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/stats" ) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 8508d7d6a700..9edba3b5b4a0 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -792,6 +792,8 @@ metricbeat.modules: include: [] exclude: [] +#------------------------------ PGBouncer Module ------------------------------ + #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm metricsets: diff --git a/metricbeat/module/pgbouncer/_meta/Dockerfile b/metricbeat/module/pgbouncer/_meta/Dockerfile new file mode 100644 index 000000000000..b820940c9273 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9.18-bullseye as build + +COPY . beats +RUN wget https://dl.google.com/go/go1.20.10.linux-amd64.tar.gz +RUN tar -C /usr/local -xzf go1.20.10.linux-amd64.tar.gz +ENV PATH=$PATH:/usr/local/go/bin +RUN mkdir -p /root/go/bin +RUN git clone https://github.com/magefile/mage +RUN cd mage; go run bootstrap.go +RUN cp /root/go/bin/mage /usr/local/go/bin/ + +WORKDIR /beats/metricbeat +RUN mage update && mage build + +FROM debian:trixie-slim AS final + +RUN mkdir -p /etc/metricbeat +COPY --from=build /beats/metricbeat/metricbeat /usr/bin/metricbeat + +ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml index ee23ba65011f..a288ffd1c680 100644 --- a/metricbeat/module/pgbouncer/_meta/config.yml +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -6,10 +6,12 @@ metricbeat.modules: metricsets: - stats period: 10s - hosts: ["pgbouncer://localhost:5432"] + hosts: ["postgresql://localhost:5432"] username: test password: password output.elasticsearch: hosts: ["http://172.21.186.195:9200"] username: elastic - password: xxxx \ No newline at end of file + password: xxxx + indices: + - index: "test" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/doc.go b/metricbeat/module/pgbouncer/doc.go index d87b920f355d..c8dc6850145e 100644 --- a/metricbeat/module/pgbouncer/doc.go +++ b/metricbeat/module/pgbouncer/doc.go @@ -16,6 +16,6 @@ // under the License. /* -Package postgresql is a Metricbeat module that contains MetricSets. +Package pgbouncer is a Metricbeat module that contains MetricSets. */ package pgbouncer \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml index e69de29bb2d1..b53723042c71 100644 --- a/metricbeat/module/pgbouncer/docker-compose.yml +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -0,0 +1,48 @@ +version: '3.9' + +services: + postgresql: + image: postgres:15 + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_USER=test + - POSTGRES_DB=test + - POSTGRES_HOST_AUTH_METHOD=md5 + - POSTGRES_INITDB_ARGS=--auth=md5 + + pgbouncer: + image: pgbouncer/pgbouncer + ports: + - 6432:6432 + environment: + - DATABASES_HOST=postgresql + - DATABASES_PORT=5432 + - DATABASES_USER=test + - DATABASES_PASSWORD=password + - DATABASES_DBNAME=test + - PGBOUNCER_LISTEN_PORT=6432 + + metricbeat: + image: test + build: + context: ./../../../../beats + dockerfile: ./metricbeat/module/pgbouncer/_meta/Dockerfile + + elasticsearch: + image: elasticsearch:8.11.0 + hostname: elasticsearch + environment: + - "ES_JAVA_OPTS=-Xms256m -Xmx256m" + - "http.host=0.0.0.0" + - "xpack.security.enabled=false" + - "discovery.type=single-node" + # - "transport.host=127.0.0.1" + ports: + - 9200:9200 + + kibana: + image: kibana:8.11.0 + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + ports: + - 5601:5601 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go new file mode 100644 index 000000000000..a51e7c1f8df6 --- /dev/null +++ b/metricbeat/module/pgbouncer/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package pgbouncer + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "pgbouncer", asset.ModuleFieldsPri, AssetPgbouncer); err != nil { + panic(err) + } +} + +// AssetPgbouncer returns asset data. +// This is the base64 encoded zlib format compressed contents of module/pgbouncer. +func AssetPgbouncer() string { + return "eJzMlMGOnDAMhu88xa89d3kADj20laoe2u0bjDzBQDSQUMcw5e2rwA7LILRSpZUmPsbw+/NvJ8+48FSgr89+cIYlA9RqywWefn//spw9ZUDJwYjt1XpX4HMGAD9ZxZoA49uWjXKJSnz3JpVnQGi86Ml4V9m6QEVt4AwQbpkCF6gpAyrLbRmKWfMZjjq+54mhUx8/Fz/0rycHQDFWaHQLXv6a2lbZVgpKGtbTo0rvVIvx4hilN0PHTtGzILCMLOjFGw7hU/Tgal0N6yovHUWN6ABFx9RDG4YZRNjpnS4ZtaPVCb6CNqQbwcE0oDCjM8iVt//xZ2CZcnxdJ3Ke7jTnfGSZu843ufuR3GJv2ta4kpTONM9zG4t/F56uXspd7h0XY/yijpdueVWHNjbgTObCroSN2+bc0pv6/BBMvVJ7mls9GT/sbL0Rtt7V/4f38uPbh9EJG7Yj7w1KAS3sFzENrL9k9KS2O963x7Itu5Yo3JVsssbNQ03vjtJYp4yW6tMW2YTNmBxUck/aumDJ3cu3/UoS7fGP2b8AAAD//5iVEwc=" +} diff --git a/metricbeat/modules.d/pgbouncer.yml.disabled b/metricbeat/modules.d/pgbouncer.yml.disabled new file mode 100644 index 000000000000..373d7902113a --- /dev/null +++ b/metricbeat/modules.d/pgbouncer.yml.disabled @@ -0,0 +1,20 @@ +# Module: pgbouncer +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html + +# Module: pgbouncer +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html + +metricbeat.modules: +- module: pgbouncer + metricsets: + - stats + period: 10s + hosts: ["postgresql://localhost:5432"] + username: test + password: password +output.elasticsearch: + hosts: ["http://172.21.186.195:9200"] + username: elastic + password: xxxx + indices: + - index: "test" \ No newline at end of file From 9754c9d6ba09027a23a6074fa2800c893970702d Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:29 +0200 Subject: [PATCH 03/53] add metricsets --- metricbeat/include/list_common.go | 3 + metricbeat/module/pgbouncer/_meta/Dockerfile | 11 ++- metricbeat/module/pgbouncer/_meta/config.yml | 4 +- .../module/pgbouncer/_meta/config_local.yml | 17 +++++ .../module/pgbouncer/docker-compose.yml | 29 ++++---- .../module/pgbouncer/lists/_meta/fields.yml | 50 +++++++++++++ metricbeat/module/pgbouncer/lists/data.go | 21 ++++++ metricbeat/module/pgbouncer/lists/lists.go | 51 ++++++++++++++ .../module/pgbouncer/mem/_meta/fields.yml | 50 +++++++++++++ metricbeat/module/pgbouncer/mem/data.go | 38 ++++++++++ metricbeat/module/pgbouncer/mem/mem.go | 47 +++++++++++++ metricbeat/module/pgbouncer/pgbouncer.go | 4 +- metricbeat/module/pgbouncer/pgbouncer.ini | 21 ++++++ .../module/pgbouncer/pools/_meta/fields.yml | 66 +++++++++++++++++ metricbeat/module/pgbouncer/pools/data.go | 48 +++++++++++++ metricbeat/module/pgbouncer/pools/pools.go | 70 +++++++++++++++++++ .../module/pgbouncer/stats/_meta/fields.yml | 40 ++++++----- metricbeat/module/pgbouncer/stats/data.go | 40 ++++++----- metricbeat/module/pgbouncer/stats/stats.go | 60 +++++++++++----- metricbeat/module/pgbouncer/userlist.txt | 3 + 20 files changed, 602 insertions(+), 71 deletions(-) create mode 100644 metricbeat/module/pgbouncer/_meta/config_local.yml create mode 100644 metricbeat/module/pgbouncer/lists/_meta/fields.yml create mode 100644 metricbeat/module/pgbouncer/lists/data.go create mode 100644 metricbeat/module/pgbouncer/lists/lists.go create mode 100644 metricbeat/module/pgbouncer/mem/_meta/fields.yml create mode 100644 metricbeat/module/pgbouncer/mem/data.go create mode 100644 metricbeat/module/pgbouncer/mem/mem.go create mode 100644 metricbeat/module/pgbouncer/pgbouncer.ini create mode 100644 metricbeat/module/pgbouncer/pools/_meta/fields.yml create mode 100644 metricbeat/module/pgbouncer/pools/data.go create mode 100644 metricbeat/module/pgbouncer/pools/pools.go create mode 100644 metricbeat/module/pgbouncer/userlist.txt diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 7b698406e07b..709fce2aec56 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -136,6 +136,9 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics" _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics/collector" _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/lists" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/mem" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/pools" _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/stats" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm/pool" diff --git a/metricbeat/module/pgbouncer/_meta/Dockerfile b/metricbeat/module/pgbouncer/_meta/Dockerfile index b820940c9273..1cbca3d30f83 100644 --- a/metricbeat/module/pgbouncer/_meta/Dockerfile +++ b/metricbeat/module/pgbouncer/_meta/Dockerfile @@ -1,7 +1,9 @@ FROM python:3.9.18-bullseye as build COPY . beats -RUN wget https://dl.google.com/go/go1.20.10.linux-amd64.tar.gz +RUN \ + --mount=type=cache,target=/var/cache/go \ + wget https://dl.google.com/go/go1.20.10.linux-amd64.tar.gz RUN tar -C /usr/local -xzf go1.20.10.linux-amd64.tar.gz ENV PATH=$PATH:/usr/local/go/bin RUN mkdir -p /root/go/bin @@ -10,11 +12,14 @@ RUN cd mage; go run bootstrap.go RUN cp /root/go/bin/mage /usr/local/go/bin/ WORKDIR /beats/metricbeat -RUN mage update && mage build +RUN \ + --mount=type=cache,target=/var/cache/mage \ + mage update && mage build FROM debian:trixie-slim AS final RUN mkdir -p /etc/metricbeat COPY --from=build /beats/metricbeat/metricbeat /usr/bin/metricbeat +COPY --from=build /beats/metricbeat/module/pgbouncer/_meta/config.yml /etc/metricbeat/metricbeat.yml -ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file +ENTRYPOINT ["metricbeat", "run", "--path.config", "/etc/metricbeat"] \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml index a288ffd1c680..258333fe9506 100644 --- a/metricbeat/module/pgbouncer/_meta/config.yml +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -6,11 +6,11 @@ metricbeat.modules: metricsets: - stats period: 10s - hosts: ["postgresql://localhost:5432"] + hosts: ["postgresql://pgbouncer:6432/pgbouncer?sslmode=disable"] username: test password: password output.elasticsearch: - hosts: ["http://172.21.186.195:9200"] + hosts: ["http://elasticsearch:9200"] username: elastic password: xxxx indices: diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml new file mode 100644 index 000000000000..80562b1ea154 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -0,0 +1,17 @@ +# Module: pgbouncer +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html + +metricbeat.modules: +- module: pgbouncer + metricsets: + - lists + period: 10s + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + username: test + password: password +output.elasticsearch: + hosts: ["http://localhost:9200"] + username: elastic + password: xxxx + indices: + - index: "test" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml index b53723042c71..07caf5f21153 100644 --- a/metricbeat/module/pgbouncer/docker-compose.yml +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -3,6 +3,7 @@ version: '3.9' services: postgresql: image: postgres:15 + hostname: postgresql environment: - POSTGRES_PASSWORD=password - POSTGRES_USER=test @@ -11,28 +12,26 @@ services: - POSTGRES_INITDB_ARGS=--auth=md5 pgbouncer: - image: pgbouncer/pgbouncer + image: edoburu/pgbouncer + hostname: pgbouncer ports: - 6432:6432 - environment: - - DATABASES_HOST=postgresql - - DATABASES_PORT=5432 - - DATABASES_USER=test - - DATABASES_PASSWORD=password - - DATABASES_DBNAME=test - - PGBOUNCER_LISTEN_PORT=6432 - - metricbeat: - image: test - build: - context: ./../../../../beats - dockerfile: ./metricbeat/module/pgbouncer/_meta/Dockerfile + volumes: + - ./pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini + - ./userlist.txt:/etc/pgbouncer/userlist.txt + # metricbeat: + # image: test + # build: + # context: ./../../../../beats + # dockerfile: ./metricbeat/module/pgbouncer/_meta/Dockerfile + # cache_from: + # - test elasticsearch: image: elasticsearch:8.11.0 hostname: elasticsearch environment: - - "ES_JAVA_OPTS=-Xms256m -Xmx256m" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - "http.host=0.0.0.0" - "xpack.security.enabled=false" - "discovery.type=single-node" diff --git a/metricbeat/module/pgbouncer/lists/_meta/fields.yml b/metricbeat/module/pgbouncer/lists/_meta/fields.yml new file mode 100644 index 000000000000..1f383003bed2 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: lists + type: group + description: > + Shows various internal information lists. + release: ga + fields: + - name: databases + type: long + description: > + Count of databases. + - name: users + type: long + description: > + Count of users. + - name: pools + type: long + description: > + Count of pools. + - name: free_clients + type: long + description: > + Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. + - name: used_clients + type: long + description: > + Count of used clients. + - name: login_clients + type: long + description: > + Count of clients in login state. + - name: free_servers + type: long + description: > + Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. + - name: used_servers + type: long + description: > + Count of used servers. + - name: dns_names + type: long + description: > + Count of DNS names in the cache. + - name: dns_zones + type: long + description: > + Count of DNS zones in the cache. + - name: dns_queries + type: long + description: > + Count of in-flight DNS queries. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go new file mode 100644 index 000000000000..4a1fcc008d5a --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -0,0 +1,21 @@ +package lists + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +var schema = s.Schema{ + "databases": mapstrstr.Int("databases"), + "users": mapstrstr.Int("users"), + "peers": mapstrstr.Int("peers"), + "pools": mapstrstr.Int("pools"), + "peer_pools": mapstrstr.Int("peer_pools"), + "free_clients": mapstrstr.Int("free_clients"), + "used_clients": mapstrstr.Int("used_clients"), + "login_clients": mapstrstr.Int("login_clients"), + "free_servers": mapstrstr.Int("free_servers"), + "used_servers": mapstrstr.Int("used_servers"), + "dns_names": mapstrstr.Int("dns_names"), + "dns_zones": mapstrstr.Int("dns_zones"), +} diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go new file mode 100644 index 000000000000..a997ac9b19b8 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -0,0 +1,51 @@ +package lists + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "lists", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +type MetricSet struct { + *pgbouncer.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + ctx := context.Background() + results, err := m.QueryStats(ctx, "SHOW LISTS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + + for _, result := range results { + data, err := schema.Apply(result) + if err != nil { + return fmt.Errorf("error applying schema: %w", err) + } + + event := mapstr.M(data) + reporter.Event(mb.Event{ + MetricSetFields: event, + }) + } + + return nil +} diff --git a/metricbeat/module/pgbouncer/mem/_meta/fields.yml b/metricbeat/module/pgbouncer/mem/_meta/fields.yml new file mode 100644 index 000000000000..1f383003bed2 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: lists + type: group + description: > + Shows various internal information lists. + release: ga + fields: + - name: databases + type: long + description: > + Count of databases. + - name: users + type: long + description: > + Count of users. + - name: pools + type: long + description: > + Count of pools. + - name: free_clients + type: long + description: > + Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. + - name: used_clients + type: long + description: > + Count of used clients. + - name: login_clients + type: long + description: > + Count of clients in login state. + - name: free_servers + type: long + description: > + Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. + - name: used_servers + type: long + description: > + Count of used servers. + - name: dns_names + type: long + description: > + Count of DNS names in the cache. + - name: dns_zones + type: long + description: > + Count of DNS zones in the cache. + - name: dns_queries + type: long + description: > + Count of in-flight DNS queries. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/mem/data.go b/metricbeat/module/pgbouncer/mem/data.go new file mode 100644 index 000000000000..99cf4174724f --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/data.go @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mem + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +// Based on pgbouncer show mem; +var schema = s.Schema{ + "databases": c.Int("databases"), + "users": c.Int("users"), + "pools": c.Int("pools"), + "free_clients": c.Int("free_clients"), + "used_clients": c.Int("used_clients"), + "login_clients": c.Int("login_clients"), + "free_servers": c.Int("free_servers"), + "used_servers": c.Int("used_servers"), + "dns_names": c.Int("dns_names"), + "dns_zones": c.Int("dns_zones"), + "dns_queries": c.Int("dns_queries"), +} diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go new file mode 100644 index 000000000000..5c69b96605b5 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -0,0 +1,47 @@ +package mem + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" +) + +// init registers the MetricSet with the central registry.// +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "mem", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New create a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + ctx := context.Background() + results, err := m.QueryStats(ctx, "SHOW MEM;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + if len(results) == 0 { + return fmt.Errorf("No results from the stats query") + } + data, _ := schema.Apply(results[0]) + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + return nil +} diff --git a/metricbeat/module/pgbouncer/pgbouncer.go b/metricbeat/module/pgbouncer/pgbouncer.go index 3b7cf25cc8ce..10fa89993d9c 100644 --- a/metricbeat/module/pgbouncer/pgbouncer.go +++ b/metricbeat/module/pgbouncer/pgbouncer.go @@ -16,7 +16,7 @@ // under the License. /* -Package pgbouncer is Metricbeat module for PostgreSQL server. +Package pgbouncer is Metricbeat module for pgbouncer pooler. */ package pgbouncer @@ -52,7 +52,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { return &base, nil } -// ParseURL is the postgres host parser +// ParseURL is the pgbouncer host parser func ParseURL(mod mb.Module, rawURL string) (mb.HostData, error) { c := struct { Username string `config:"username"` diff --git a/metricbeat/module/pgbouncer/pgbouncer.ini b/metricbeat/module/pgbouncer/pgbouncer.ini new file mode 100644 index 000000000000..c8826244eb43 --- /dev/null +++ b/metricbeat/module/pgbouncer/pgbouncer.ini @@ -0,0 +1,21 @@ +[databases] +* = host=postgresql port=5432 dbname=test + +[users] +# dostosować wartości max_user_connections do wielkości bazy +test = pool_mode=statement max_user_connections=40 +user1 = pool_mode=transaction max_user_connections=25 +user2 = pool_mode=transaction max_user_connections=25 + +[pgbouncer] +listen_addr = 0.0.0.0 +listen_port = 6432 +auth_type = trust +admin_users = test +auth_file = /etc/pgbouncer/userlist.txt +max_client_conn = 8000 +default_pool_size = 400 +server_idle_timeout = 120 + +ignore_startup_parameters = extra_float_digits +log_connections = 0 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/_meta/fields.yml b/metricbeat/module/pgbouncer/pools/_meta/fields.yml new file mode 100644 index 000000000000..7396b37872ac --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/fields.yml @@ -0,0 +1,66 @@ +- name: pools + type: group + description: > + Shows the current state of the connection pools. + release: ga + fields: + - name: database + type: keyword + description: > + Name of the database. + - name: user + type: keyword + description: > + Name of the user. + - name: cl_active + type: long + description: > + Client connections that are either linked to server connections or are idle with no queries waiting to be processed. + - name: cl_waiting + type: long + description: > + Client connections that have sent queries but have not yet got a server connection. + - name: cl_active_cancel_req + type: long + description: > + Client connections that have forwarded query cancellations to the server and are waiting for the server response. + - name: cl_waiting_cancel_req + type: long + description: > + Client connections that have not forwarded query cancellations to the server yet. + - name: sv_active + type: long + description: > + Server connections that are linked to a client. + - name: sv_active_cancel + type: long + description: > + Server connections that are currently forwarding a cancel request. + - name: sv_being_canceled + type: long + description: > + Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. + - name: sv_idle + type: long + description: > + Server connections that are unused and immediately usable for client queries. + - name: sv_used + type: long + description: > + Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. + - name: sv_tested + type: long + description: > + Server connections that are currently running either server_reset_query or server_check_query. + - name: sv_login + type: long + description: > + Server connections currently in the process of logging in. + - name: maxwait_us + type: long + description: > + Microsecond part of the maximum waiting time. Represents the total wait time in microseconds. + - name: pool_mode + type: keyword + description: > + The pooling mode in use. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/data.go b/metricbeat/module/pgbouncer/pools/data.go new file mode 100644 index 000000000000..fbaae9a479c5 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/data.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pools + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// Based on pgbouncer show pools; +var schema = s.Schema{ + "database": c.Str("database"), + "user": c.Str("user"), + "cl_active": c.Int("cl_active"), + "cl_waiting": c.Int("cl_waiting"), + "cl_active_cancel_req": c.Int("cl_active_cancel_req"), + "cl_waiting_cancel_req": c.Int("cl_waiting_cancel_req"), + "sv_active": c.Int("sv_active"), + "sv_active_cancel": c.Int("sv_active_cancel"), + "sv_being_canceled": c.Int("sv_being_canceled"), + "sv_idle": c.Int("sv_idle"), + "sv_used": c.Int("sv_used"), + "sv_tested": c.Int("sv_tested"), + "sv_login": c.Int("sv_login"), + "maxwait_us": c.Int("maxwait_us"), + "pool_mode": c.Str("pool_mode"), +} + +// MapResult maps a single result to a mapstr.M +func MapResult(result map[string]interface{}) (mapstr.M, error) { + return schema.Apply(result) +} diff --git a/metricbeat/module/pgbouncer/pools/pools.go b/metricbeat/module/pgbouncer/pools/pools.go new file mode 100644 index 000000000000..a26b050f809d --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/pools.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pools + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" +) + +// init registers the MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "pools", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + ctx := context.Background() + results, err := m.QueryStats(ctx, "SHOW POOLS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + + for _, result := range results { + event, err := MapResult(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + reporter.Event(mb.Event{ + MetricSetFields: event, + }) + } + + return nil +} diff --git a/metricbeat/module/pgbouncer/stats/_meta/fields.yml b/metricbeat/module/pgbouncer/stats/_meta/fields.yml index 8836ce25ddc2..94fc307392fb 100644 --- a/metricbeat/module/pgbouncer/stats/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/stats/_meta/fields.yml @@ -1,9 +1,7 @@ - name: stats type: group description: > - One document per server process, showing information related to the current - activity of that process, such as state and current query. Collected by - querying stats. + Shows statistics since the process start. release: ga fields: - name: database @@ -13,56 +11,64 @@ - name: total_query_count type: long description: > - OID of the database this backend is connected to. + Total number of SQL commands pooled by pgbouncer. + - name: total_server_assignment_count + type: long + description: > + Total times a server was assigned to a client. - name: total_received type: long description: > - OID of the database this backend is connected to. + Total volume in bytes of network traffic received by pgbouncer. - name: total_sent type: long description: > - OID of the database this backend is connected to. + Total volume in bytes of network traffic sent by pgbouncer. - name: total_xact_time type: long description: > - OID of the database this backend is connected to. + Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. - name: total_query_time type: long description: > - OID of the database this backend is connected to. + Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. - name: total_wait_time type: long description: > - OID of the database this backend is connected to. + Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. - name: total_xact_count type: long description: > - OID of the database this backend is connected to. + Total number of SQL transactions pooled by pgbouncer. - name: avg_xact_count type: long description: > - OID of the database this backend is connected to. + Average transactions per second in last stat period. - name: avg_query_count type: long description: > - OID of the database this backend is connected to. + Average queries per second in last stat period. + - name: avg_server_assignment_count + type: long + description: > + Average number of times a server as assigned to a client per second in the last stat period. - name: avg_recv type: long description: > - OID of the database this backend is connected to. + Average received (from clients) bytes per second. - name: avg_sent type: long description: > - OID of the database this backend is connected to. + Average sent (to clients) bytes per second. - name: avg_xact_time type: long description: > - OID of the database this backend is connected to. + Average transaction duration, in microseconds. - name: avg_query_time type: long description: > - OID of the database this backend is connected to. + Average query duration, in microseconds. - name: avg_wait_time type: long description: > - OID of the database this backend is connected to. \ No newline at end of file + Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go index b03f76e3f094..3e29e0455ab6 100644 --- a/metricbeat/module/pgbouncer/stats/data.go +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -20,23 +20,31 @@ package stats import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" + "github.com/elastic/elastic-agent-libs/mapstr" ) // Based on pgbouncer show stats; var schema = s.Schema{ - "database": c.Str("database"), - "total_query_count": c.Int("total_query_count"), - "total_received": c.Int("total_received"), - "total_sent": c.Int("total_sent"), - "total_xact_time": c.Int("total_xact_time"), - "total_query_time": c.Int("total_query_time"), - "total_wait_time": c.Int("total_wait_time"), - "total_xact_count": c.Int("total_xact_count"), - "avg_xact_count": c.Int("avg_xact_count"), - "avg_query_count": c.Int("avg_query_count"), - "avg_recv": c.Int("avg_recv"), - "avg_sent": c.Int("avg_sent"), - "avg_xact_time": c.Int("avg_xact_time"), - "avg_query_time": c.Int("avg_query_time"), - "avg_wait_time": c.Int("avg_wait_time"), -} \ No newline at end of file + "database": c.Str("database"), + "total_query_count": c.Int("total_query_count"), + "total_server_assignment_count": c.Int("total_server_assignment_count"), + "total_received": c.Int("total_received"), + "total_sent": c.Int("total_sent"), + "total_xact_time": c.Int("total_xact_time"), + "total_query_time": c.Int("total_query_time"), + "total_wait_time": c.Int("total_wait_time"), + "total_xact_count": c.Int("total_xact_count"), + "avg_xact_count": c.Int("avg_xact_count"), + "avg_query_count": c.Int("avg_query_count"), + "avg_server_assignment_count": c.Int("avg_server_assignment_count"), + "avg_recv": c.Int("avg_recv"), + "avg_sent": c.Int("avg_sent"), + "avg_xact_time": c.Int("avg_xact_time"), + "avg_query_time": c.Int("avg_query_time"), + "avg_wait_time": c.Int("avg_wait_time"), +} + +// MapResult maps a single result to a mapstr.M +func MapResult(result map[string]interface{}) (mapstr.M, error) { + return schema.Apply(result) +} diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go index 3bbc83fdef8b..40a7fa269ab3 100644 --- a/metricbeat/module/pgbouncer/stats/stats.go +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -1,42 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package stats + import ( "context" "fmt" + "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" ) + // init registers the MetricSet with the central registry. -// The New method will be called after the setup of the module and before starting to fetch data func init() { - mb.Registry.MustAddMetricSet("pgbouncer", "stats", New, - mb.WithHostParser(postgresql.ParseURL), - mb.DefaultMetricSet(), - ) + mb.Registry.MustAddMetricSet("pgbouncer", "stats", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) } + // MetricSet type defines all fields of the MetricSet type MetricSet struct { - *postgresql.MetricSet + *pgbouncer.MetricSet } -// New create a new instance of the MetricSet. + +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - ms, err := postgresql.NewMetricSet(base) + ms, err := pgbouncer.NewMetricSet(base) if err != nil { return nil, err } return &MetricSet{MetricSet: ms}, nil } + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { ctx := context.Background() results, err := m.QueryStats(ctx, "SHOW STATS;") if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } - if len(results) == 0 { - return fmt.Errorf("No results from the stats query") + + for _, result := range results { + event, err := MapResult(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + reporter.Event(mb.Event{ + MetricSetFields: event, + }) } - data, _ := schema.Apply(results[0]) - reporter.Event(mb.Event{ - MetricSetFields: data, - }) + return nil -} \ No newline at end of file +} diff --git a/metricbeat/module/pgbouncer/userlist.txt b/metricbeat/module/pgbouncer/userlist.txt new file mode 100644 index 000000000000..a8545ab8caa8 --- /dev/null +++ b/metricbeat/module/pgbouncer/userlist.txt @@ -0,0 +1,3 @@ +"test" "password" +"user1" "password" +"user2" "password" \ No newline at end of file From 54bbcc7a603b4bf189f063cef961ef227cddec3b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:30 +0200 Subject: [PATCH 04/53] fix list metricset --- metricbeat/module/pgbouncer/lists/data.go | 31 +++++++++------- metricbeat/module/pgbouncer/lists/lists.go | 42 ++++++++++++++++------ 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 4a1fcc008d5a..1b488c7ab88c 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -2,20 +2,25 @@ package lists import ( s "github.com/elastic/beats/v7/libbeat/common/schema" - "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/elastic-agent-libs/mapstr" ) var schema = s.Schema{ - "databases": mapstrstr.Int("databases"), - "users": mapstrstr.Int("users"), - "peers": mapstrstr.Int("peers"), - "pools": mapstrstr.Int("pools"), - "peer_pools": mapstrstr.Int("peer_pools"), - "free_clients": mapstrstr.Int("free_clients"), - "used_clients": mapstrstr.Int("used_clients"), - "login_clients": mapstrstr.Int("login_clients"), - "free_servers": mapstrstr.Int("free_servers"), - "used_servers": mapstrstr.Int("used_servers"), - "dns_names": mapstrstr.Int("dns_names"), - "dns_zones": mapstrstr.Int("dns_zones"), + "databases": c.Int("databases"), + "users": c.Int("users"), + "peers": c.Int("peers"), + "pools": c.Int("pools"), + "peer_pools": c.Int("peer_pools"), + "free_clients": c.Int("free_clients"), + "used_clients": c.Int("used_clients"), + "login_clients": c.Int("login_clients"), + "free_servers": c.Int("free_servers"), + "used_servers": c.Int("used_servers"), + "dns_names": c.Int("dns_names"), + "dns_zones": c.Int("dns_zones"), +} + +func MapResult(result map[string]interface{}) (mapstr.M, error) { + return schema.Apply(result) } diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index a997ac9b19b8..5d0416cd3b94 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -3,12 +3,13 @@ package lists import ( "context" "fmt" + "strconv" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" - "github.com/elastic/elastic-agent-libs/mapstr" ) +// init registers the MetricSet with the central registry. func init() { mb.Registry.MustAddMetricSet("pgbouncer", "lists", New, mb.WithHostParser(pgbouncer.ParseURL), @@ -16,10 +17,12 @@ func init() { ) } +// MetricSet type defines all fields of the MetricSet type MetricSet struct { *pgbouncer.MetricSet } +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := pgbouncer.NewMetricSet(base) if err != nil { @@ -28,24 +31,41 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{MetricSet: ms}, nil } +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { ctx := context.Background() results, err := m.QueryStats(ctx, "SHOW LISTS;") + fmt.Printf("\nPOOLS PRINTING results: %v\n\n", results) if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } - - for _, result := range results { - data, err := schema.Apply(result) - if err != nil { - return fmt.Errorf("error applying schema: %w", err) + resultMap := make(map[string]interface{}) + for _, s := range results { + key := s["list"].(string) + // Convert value to int if necessary + var value int + switch v := s["items"].(type) { + case string: + value, err = strconv.Atoi(v) + if err != nil { + return fmt.Errorf("error converting value to int: %w", err) + } + case int: + value = v + default: + return fmt.Errorf("unexpected type for value: %T", v) } - event := mapstr.M(data) - reporter.Event(mb.Event{ - MetricSetFields: event, - }) + // Assign the integer value + resultMap[key] = value } - + event, err := MapResult(resultMap) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + reporter.Event(mb.Event{ + MetricSetFields: event, + }) return nil } From 0d688c20173a1ecad4d786e4b6f2d22ae0baf882 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:30 +0200 Subject: [PATCH 05/53] fix --- metricbeat/module/pgbouncer/_meta/config.yml | 2 ++ metricbeat/module/pgbouncer/fields.go | 2 +- metricbeat/module/pgbouncer/lists/data.go | 2 +- metricbeat/module/pgbouncer/lists/lists.go | 25 ++++++++------------ 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml index 258333fe9506..15e1e260d56c 100644 --- a/metricbeat/module/pgbouncer/_meta/config.yml +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -5,6 +5,8 @@ metricbeat.modules: - module: pgbouncer metricsets: - stats + - lists + - pools period: 10s hosts: ["postgresql://pgbouncer:6432/pgbouncer?sslmode=disable"] username: test diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go index a51e7c1f8df6..d7d5175a42f7 100644 --- a/metricbeat/module/pgbouncer/fields.go +++ b/metricbeat/module/pgbouncer/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPgbouncer returns asset data. // This is the base64 encoded zlib format compressed contents of module/pgbouncer. func AssetPgbouncer() string { - return "eJzMlMGOnDAMhu88xa89d3kADj20laoe2u0bjDzBQDSQUMcw5e2rwA7LILRSpZUmPsbw+/NvJ8+48FSgr89+cIYlA9RqywWefn//spw9ZUDJwYjt1XpX4HMGAD9ZxZoA49uWjXKJSnz3JpVnQGi86Ml4V9m6QEVt4AwQbpkCF6gpAyrLbRmKWfMZjjq+54mhUx8/Fz/0rycHQDFWaHQLXv6a2lbZVgpKGtbTo0rvVIvx4hilN0PHTtGzILCMLOjFGw7hU/Tgal0N6yovHUWN6ABFx9RDG4YZRNjpnS4ZtaPVCb6CNqQbwcE0oDCjM8iVt//xZ2CZcnxdJ3Ke7jTnfGSZu843ufuR3GJv2ta4kpTONM9zG4t/F56uXspd7h0XY/yijpdueVWHNjbgTObCroSN2+bc0pv6/BBMvVJ7mls9GT/sbL0Rtt7V/4f38uPbh9EJG7Yj7w1KAS3sFzENrL9k9KS2O963x7Itu5Yo3JVsssbNQ03vjtJYp4yW6tMW2YTNmBxUck/aumDJ3cu3/UoS7fGP2b8AAAD//5iVEwc=" + return "eJzsWs+O2zYTv+9TDHJKgI0fYA8f8LUFemmCtJuehRE1lomlOA5J2as+fTEkJcteebPeSpsWsE8BSc385u9vyOxHeKDuDrZ1ya1V5G4Agg6G7uDdl19/SmvvbgAq8srpbdBs7+B/NwAAnyg4rTwoNoZUoArWjpuDqNUNgN+wC4Viu9b1HazReLoBcGQIPd1BjTcAa02m8ndR5kew2NAxHvmFbivHHbfbvDIBSH4DaGgSvFXeGmsZazLaBz+sTml6Rpv87je897BDp7n1oG0gZ9GAtmt2DcoXScdq9NWxA/rfKcQxzAoDlujJH+32cA3b+mTjGcTy+5lbG4DXB8GrSb2tJze7zih0Wt+W2cyuLwqd1rd2RIUymmyYXa3Ihix7BV835AnQDUsQNhjiQqW9YmtjEd1C2Qb4Uvdp/EC0laMEDTXsOkDHra3Sx3v0gMawwlh+7ORgA55BB1BooSRw1Pq8uW5DO9bPgDvWVS9Csz2fBdVSXoroei9Najdca7uU+t4Z2iY94AMGeiZZPLndAjURkyXLHidLXvphyTLovyxZFvJSRNd7abpRWl/Iv2ZX/cvn+6gipoq4WKHanEkUAfEX22VARMEvBPGtJafnh6Htx7XR9SZEQFnJ6sqtJ266cuuVW6/ceuXWK7deuXVubj1t5q/k1mhn6xzZkKpTIMTFVAdCsU9a/OspdtJZD9Tt2VWX+eszNgPWXvp5ml1Gr0ie1qlMgSro3bTBr8iO2ElHQRk1LdJhQw6Mtg9USS9JNXx0mF08qitDsNdhA5b71II96qBtLV+WBFvHiryn6qxh+fzClm1wJw3ShgGntOO4ajlARwFqDoBPrf1ORAqFVpEpHH17CxPW7PboKqqiHR0k7QbzOY6JlG1AW8Uw9RHJPNJvO/JbtufS/BCZN7ZQwnGJlR2FaQv8bt6iuX9aB0PRHKoF85jyHUzZqW8ALbdj0/VelVTA7FJw9K0lfx5tSYcMoNPe9s/gZoxW7l7GdKC4NRWUpLjJrUVKdJzAgaFimYJaG7SR8WZEb8cG+ZRMiputIRmi0lxFLneBwP0HmHOMhaW1z2l11iEC7A2i1to4P0kJ66ahSmMg00HrsTSxC+Q0e8roJ3hFzMJ4o6dLIpuiJuAadiSbNruzUBtSD0VFBrtbiWDYUAeWhhExH0ihCAyutSki1EBJ6ySOun74Tc6pUZ9pz35XBPJhcdOPC8y11kqeZgrNljnyFLJl7CbsPWtCvOUsZ8EBeJ5QM1nLPGK4rsWUcw5u8FGqsmjnGlg/aeXYk2JbwRZd6IeiBh910zaHHqAbWsEftBW3pqs5QeCAJh6J+2JPc5D3zPtF0XA14xT5VZzIbASoSBYg7Zhhh+AGnOMpS8RoH7Ty4LVVdBRFH9CF/8KcnfpuieqBpN15GC7NEHg6djHgqXoKJVedmbLwa0wk2zYlOYF5//tvwiEN2srHyFIFZXf8n5Ln0OVCR+91bRuyYQGkku3+MLXGd4Oo7yXDSILpSJHezdYpE64dmzbVYdkFih3FUtize4DgcL3WCnq9F/hzZue9AGScFV4K8BFVKCQiCyXjuKWB355Cg/2G7FHtwBf2oXYkWawtoJhlPcb2f9uzVORsIYDDntAUPZJqY8d9dsIYF+KPNj1N1nGSnPTB7WVGRX6b0yZhpgF7/5g5vpz1dXz7hMDgz20V3wKTof3wN3pV0aPKx6GZfu8aO0rcpdvoKL8uaKW4q+eH9/8dOazpBFKc2OL8oS0Y9OkFS9Y1n3m+EHTzk1APr3+peC2yZQmoR3mI8gkZneGiE3NkEHi5SY7Ubmb8Aw+9j39vlOvyQ+aEA9jn/Dy7UyPvvJdL6uVw5qahiWqBqnWYWORFo/ahThbAlW5Wr0D0L2rw8B6zNXk2Hu4yfnTZn2zxVevivejk5d0XqZY+rG7+DgAA///v/CY+" } diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 1b488c7ab88c..acc509626a56 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -2,7 +2,7 @@ package lists import ( s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 5d0416cd3b94..3e21c7855d99 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -3,7 +3,8 @@ package lists import ( "context" "fmt" - "strconv" + "log" + "os" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" @@ -11,6 +12,8 @@ import ( // init registers the MetricSet with the central registry. func init() { + log.SetOutput(os.Stderr) + log.SetFlags(0) mb.Registry.MustAddMetricSet("pgbouncer", "lists", New, mb.WithHostParser(pgbouncer.ParseURL), mb.DefaultMetricSet(), @@ -43,29 +46,21 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { resultMap := make(map[string]interface{}) for _, s := range results { key := s["list"].(string) - // Convert value to int if necessary - var value int - switch v := s["items"].(type) { - case string: - value, err = strconv.Atoi(v) - if err != nil { - return fmt.Errorf("error converting value to int: %w", err) - } - case int: - value = v - default: - return fmt.Errorf("unexpected type for value: %T", v) - } + value := s["items"] - // Assign the integer value + // Assign the value from "items" resultMap[key] = value } + fmt.Printf("\nPOOLS PRINTING resultMap: %v\n", resultMap) event, err := MapResult(resultMap) if err != nil { return fmt.Errorf("error mapping result: %w", err) } + fmt.Printf("\nPOOLS PRINTING mapped event: %v\n\n", event) + reporter.Event(mb.Event{ MetricSetFields: event, }) + return nil } From 21a248c0bb372899cef6aba19b57ab58921fdccd Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:30 +0200 Subject: [PATCH 06/53] add mem metricset --- .../module/pgbouncer/_meta/config_local.yml | 2 +- metricbeat/module/pgbouncer/lists/lists.go | 3 - metricbeat/module/pgbouncer/mem/data.go | 95 +++++++++++++------ metricbeat/module/pgbouncer/mem/mem.go | 28 ++++-- metricbeat/module/pgbouncer/todo | 10 ++ 5 files changed, 96 insertions(+), 42 deletions(-) create mode 100644 metricbeat/module/pgbouncer/todo diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index 80562b1ea154..c825b91dab36 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -4,7 +4,7 @@ metricbeat.modules: - module: pgbouncer metricsets: - - lists + - mem period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] username: test diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 3e21c7855d99..53321bbaf6a8 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -39,7 +39,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { ctx := context.Background() results, err := m.QueryStats(ctx, "SHOW LISTS;") - fmt.Printf("\nPOOLS PRINTING results: %v\n\n", results) if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } @@ -51,12 +50,10 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { // Assign the value from "items" resultMap[key] = value } - fmt.Printf("\nPOOLS PRINTING resultMap: %v\n", resultMap) event, err := MapResult(resultMap) if err != nil { return fmt.Errorf("error mapping result: %w", err) } - fmt.Printf("\nPOOLS PRINTING mapped event: %v\n\n", event) reporter.Event(mb.Event{ MetricSetFields: event, diff --git a/metricbeat/module/pgbouncer/mem/data.go b/metricbeat/module/pgbouncer/mem/data.go index 99cf4174724f..7bd54733c207 100644 --- a/metricbeat/module/pgbouncer/mem/data.go +++ b/metricbeat/module/pgbouncer/mem/data.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - package mem import ( @@ -22,17 +5,71 @@ import ( c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" ) -// Based on pgbouncer show mem; var schema = s.Schema{ - "databases": c.Int("databases"), - "users": c.Int("users"), - "pools": c.Int("pools"), - "free_clients": c.Int("free_clients"), - "used_clients": c.Int("used_clients"), - "login_clients": c.Int("login_clients"), - "free_servers": c.Int("free_servers"), - "used_servers": c.Int("used_servers"), - "dns_names": c.Int("dns_names"), - "dns_zones": c.Int("dns_zones"), - "dns_queries": c.Int("dns_queries"), + "user_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "credentials_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "db_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "peer_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "peer_pool_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "pool_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "outstanding_request_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "server_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "iobuf_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "var_list_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "server_prepared_statement_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, } diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index 5c69b96605b5..94cd9d39eef5 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -3,14 +3,18 @@ package mem import ( "context" "fmt" + "log" + "os" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" ) -// init registers the MetricSet with the central registry.// -// The New method will be called after the setup of the module and before starting to fetch data +// init registers the MetricSet with the central registry. func init() { + log.SetOutput(os.Stderr) + log.SetFlags(0) mb.Registry.MustAddMetricSet("pgbouncer", "mem", New, mb.WithHostParser(pgbouncer.ParseURL), mb.DefaultMetricSet(), @@ -22,7 +26,7 @@ type MetricSet struct { *pgbouncer.MetricSet } -// New create a new instance of the MetricSet. +// New creates a new instance of the MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { ms, err := pgbouncer.NewMetricSet(base) if err != nil { @@ -30,18 +34,24 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } return &MetricSet{MetricSet: ms}, nil } + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { ctx := context.Background() results, err := m.QueryStats(ctx, "SHOW MEM;") if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } - if len(results) == 0 { - return fmt.Errorf("No results from the stats query") + + for _, result := range results { + var data mapstr.M + + data, _ = schema.Apply(result) + + reporter.Event(mb.Event{ + MetricSetFields: data, + }) } - data, _ := schema.Apply(results[0]) - reporter.Event(mb.Event{ - MetricSetFields: data, - }) return nil } diff --git a/metricbeat/module/pgbouncer/todo b/metricbeat/module/pgbouncer/todo new file mode 100644 index 000000000000..e9c0616cebb6 --- /dev/null +++ b/metricbeat/module/pgbouncer/todo @@ -0,0 +1,10 @@ + + +ogarnąć typy (ms, s itd) + +dostosowac konfiguracje metricbeat dla pgbouncer + +usunac jakos connection string z logow + +napisac testy + From 4731009967a9aefe8d57ec2f58fc0a23ce697d21 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:30 +0200 Subject: [PATCH 07/53] add mem metricset --- metricbeat/module/pgbouncer/lists/data.go | 5 ----- metricbeat/module/pgbouncer/lists/lists.go | 22 ++++++++-------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index acc509626a56..3ea03c6944e9 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -3,7 +3,6 @@ package lists import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" - "github.com/elastic/elastic-agent-libs/mapstr" ) var schema = s.Schema{ @@ -20,7 +19,3 @@ var schema = s.Schema{ "dns_names": c.Int("dns_names"), "dns_zones": c.Int("dns_zones"), } - -func MapResult(result map[string]interface{}) (mapstr.M, error) { - return schema.Apply(result) -} diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 53321bbaf6a8..bcf5300d1069 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -8,6 +8,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" ) // init registers the MetricSet with the central registry. @@ -42,22 +43,15 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } - resultMap := make(map[string]interface{}) - for _, s := range results { - key := s["list"].(string) - value := s["items"] - // Assign the value from "items" - resultMap[key] = value - } - event, err := MapResult(resultMap) - if err != nil { - return fmt.Errorf("error mapping result: %w", err) - } + for _, result := range results { + var data mapstr.M - reporter.Event(mb.Event{ - MetricSetFields: event, - }) + data, _ = schema.Apply(result) + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + } return nil } From ab504d145eeb2cd48184cb310cf23ef20ef4db2f Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:31 +0200 Subject: [PATCH 08/53] add mem metricset --- .../module/pgbouncer/_meta/config_local.yml | 2 +- metricbeat/module/pgbouncer/lists/data.go | 5 +++++ metricbeat/module/pgbouncer/lists/lists.go | 22 ++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index c825b91dab36..80562b1ea154 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -4,7 +4,7 @@ metricbeat.modules: - module: pgbouncer metricsets: - - mem + - lists period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] username: test diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 3ea03c6944e9..acc509626a56 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -3,6 +3,7 @@ package lists import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" + "github.com/elastic/elastic-agent-libs/mapstr" ) var schema = s.Schema{ @@ -19,3 +20,7 @@ var schema = s.Schema{ "dns_names": c.Int("dns_names"), "dns_zones": c.Int("dns_zones"), } + +func MapResult(result map[string]interface{}) (mapstr.M, error) { + return schema.Apply(result) +} diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index bcf5300d1069..53321bbaf6a8 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -8,7 +8,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" - "github.com/elastic/elastic-agent-libs/mapstr" ) // init registers the MetricSet with the central registry. @@ -43,15 +42,22 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if err != nil { return fmt.Errorf("error in QueryStats: %w", err) } + resultMap := make(map[string]interface{}) + for _, s := range results { + key := s["list"].(string) + value := s["items"] - for _, result := range results { - var data mapstr.M + // Assign the value from "items" + resultMap[key] = value + } + event, err := MapResult(resultMap) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } - data, _ = schema.Apply(result) + reporter.Event(mb.Event{ + MetricSetFields: event, + }) - reporter.Event(mb.Event{ - MetricSetFields: data, - }) - } return nil } From 5cfe4f4278aa4ad6937b07add6835dc8acd6816c Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:31 +0200 Subject: [PATCH 09/53] simplify metricsets --- metricbeat/module/pgbouncer/_meta/config_local.yml | 2 ++ metricbeat/module/pgbouncer/pools/data.go | 6 ------ metricbeat/module/pgbouncer/pools/pools.go | 12 ++++++------ metricbeat/module/pgbouncer/stats/data.go | 6 ------ metricbeat/module/pgbouncer/stats/stats.go | 12 ++++++------ 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index 80562b1ea154..9d60974a4c77 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -4,6 +4,8 @@ metricbeat.modules: - module: pgbouncer metricsets: + - pools + - stats - lists period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] diff --git a/metricbeat/module/pgbouncer/pools/data.go b/metricbeat/module/pgbouncer/pools/data.go index fbaae9a479c5..f769bc85cc08 100644 --- a/metricbeat/module/pgbouncer/pools/data.go +++ b/metricbeat/module/pgbouncer/pools/data.go @@ -20,7 +20,6 @@ package pools import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" - "github.com/elastic/elastic-agent-libs/mapstr" ) // Based on pgbouncer show pools; @@ -41,8 +40,3 @@ var schema = s.Schema{ "maxwait_us": c.Int("maxwait_us"), "pool_mode": c.Str("pool_mode"), } - -// MapResult maps a single result to a mapstr.M -func MapResult(result map[string]interface{}) (mapstr.M, error) { - return schema.Apply(result) -} diff --git a/metricbeat/module/pgbouncer/pools/pools.go b/metricbeat/module/pgbouncer/pools/pools.go index a26b050f809d..3f9d599641a1 100644 --- a/metricbeat/module/pgbouncer/pools/pools.go +++ b/metricbeat/module/pgbouncer/pools/pools.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" ) // init registers the MetricSet with the central registry. @@ -57,14 +58,13 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } for _, result := range results { - event, err := MapResult(result) - if err != nil { - return fmt.Errorf("error mapping result: %w", err) - } + var data mapstr.M + + data, _ = schema.Apply(result) + reporter.Event(mb.Event{ - MetricSetFields: event, + MetricSetFields: data, }) } - return nil } diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go index 3e29e0455ab6..92a2967a45a0 100644 --- a/metricbeat/module/pgbouncer/stats/data.go +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -20,7 +20,6 @@ package stats import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" - "github.com/elastic/elastic-agent-libs/mapstr" ) // Based on pgbouncer show stats; @@ -43,8 +42,3 @@ var schema = s.Schema{ "avg_query_time": c.Int("avg_query_time"), "avg_wait_time": c.Int("avg_wait_time"), } - -// MapResult maps a single result to a mapstr.M -func MapResult(result map[string]interface{}) (mapstr.M, error) { - return schema.Apply(result) -} diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go index 40a7fa269ab3..05594abcc8c5 100644 --- a/metricbeat/module/pgbouncer/stats/stats.go +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" ) // init registers the MetricSet with the central registry. @@ -57,14 +58,13 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } for _, result := range results { - event, err := MapResult(result) - if err != nil { - return fmt.Errorf("error mapping result: %w", err) - } + var data mapstr.M + + data, _ = schema.Apply(result) + reporter.Event(mb.Event{ - MetricSetFields: event, + MetricSetFields: data, }) } - return nil } From 182af1d0685541c700c405cb4bcd187c3986b41b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:31 +0200 Subject: [PATCH 10/53] simplify metricsets --- metricbeat/module/pgbouncer/_meta/config_local.yml | 2 -- metricbeat/module/pgbouncer/lists/data.go | 5 ----- metricbeat/module/pgbouncer/lists/lists.go | 4 ++-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index 9d60974a4c77..80562b1ea154 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -4,8 +4,6 @@ metricbeat.modules: - module: pgbouncer metricsets: - - pools - - stats - lists period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index acc509626a56..3ea03c6944e9 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -3,7 +3,6 @@ package lists import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" - "github.com/elastic/elastic-agent-libs/mapstr" ) var schema = s.Schema{ @@ -20,7 +19,3 @@ var schema = s.Schema{ "dns_names": c.Int("dns_names"), "dns_zones": c.Int("dns_zones"), } - -func MapResult(result map[string]interface{}) (mapstr.M, error) { - return schema.Apply(result) -} diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 53321bbaf6a8..618192409fca 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -50,13 +50,13 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { // Assign the value from "items" resultMap[key] = value } - event, err := MapResult(resultMap) + data, err := schema.Apply(resultMap) if err != nil { return fmt.Errorf("error mapping result: %w", err) } reporter.Event(mb.Event{ - MetricSetFields: event, + MetricSetFields: data, }) return nil From ad0a6536bae0539bccdb130681a3037ca2a0f4ec Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:31 +0200 Subject: [PATCH 11/53] add meta --- metricbeat/module/pgbouncer/_meta/Dockerfile | 25 --- .../{ => _meta/compose}/docker-compose.yml | 7 - .../{ => _meta/compose}/pgbouncer.ini | 0 .../pgbouncer/_meta/compose/userlist.txt | 1 + .../pgbouncer/_meta/config.reference.yml | 32 ++++ metricbeat/module/pgbouncer/_meta/config.yml | 24 +-- .../module/pgbouncer/_meta/config_local.yml | 5 +- .../module/pgbouncer/_meta/docs.asciidoc | 51 +++++ .../module/pgbouncer/lists/_meta/data.json | 32 ++++ .../pgbouncer/lists/_meta/docs.asciidoc | 1 + .../module/pgbouncer/mem/_meta/data.json | 86 +++++++++ .../module/pgbouncer/mem/_meta/docs.asciidoc | 1 + .../module/pgbouncer/mem/_meta/fields.yml | 180 +++++++++++++++--- .../module/pgbouncer/pools/_meta/data.json | 35 ++++ .../pgbouncer/pools/_meta/docs.asciidoc | 1 + .../module/pgbouncer/stats/_meta/data.json | 37 ++++ .../pgbouncer/stats/_meta/docs.asciidoc | 1 + metricbeat/module/pgbouncer/stats/data.go | 12 +- metricbeat/module/pgbouncer/todo | 10 - metricbeat/module/pgbouncer/userlist.txt | 3 - 20 files changed, 449 insertions(+), 95 deletions(-) delete mode 100644 metricbeat/module/pgbouncer/_meta/Dockerfile rename metricbeat/module/pgbouncer/{ => _meta/compose}/docker-compose.yml (83%) rename metricbeat/module/pgbouncer/{ => _meta/compose}/pgbouncer.ini (100%) create mode 100644 metricbeat/module/pgbouncer/_meta/compose/userlist.txt create mode 100644 metricbeat/module/pgbouncer/_meta/docs.asciidoc create mode 100644 metricbeat/module/pgbouncer/lists/_meta/data.json create mode 100644 metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc create mode 100644 metricbeat/module/pgbouncer/mem/_meta/data.json create mode 100644 metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc create mode 100644 metricbeat/module/pgbouncer/pools/_meta/data.json create mode 100644 metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc create mode 100644 metricbeat/module/pgbouncer/stats/_meta/data.json create mode 100644 metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc delete mode 100644 metricbeat/module/pgbouncer/todo delete mode 100644 metricbeat/module/pgbouncer/userlist.txt diff --git a/metricbeat/module/pgbouncer/_meta/Dockerfile b/metricbeat/module/pgbouncer/_meta/Dockerfile deleted file mode 100644 index 1cbca3d30f83..000000000000 --- a/metricbeat/module/pgbouncer/_meta/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.9.18-bullseye as build - -COPY . beats -RUN \ - --mount=type=cache,target=/var/cache/go \ - wget https://dl.google.com/go/go1.20.10.linux-amd64.tar.gz -RUN tar -C /usr/local -xzf go1.20.10.linux-amd64.tar.gz -ENV PATH=$PATH:/usr/local/go/bin -RUN mkdir -p /root/go/bin -RUN git clone https://github.com/magefile/mage -RUN cd mage; go run bootstrap.go -RUN cp /root/go/bin/mage /usr/local/go/bin/ - -WORKDIR /beats/metricbeat -RUN \ - --mount=type=cache,target=/var/cache/mage \ - mage update && mage build - -FROM debian:trixie-slim AS final - -RUN mkdir -p /etc/metricbeat -COPY --from=build /beats/metricbeat/metricbeat /usr/bin/metricbeat -COPY --from=build /beats/metricbeat/module/pgbouncer/_meta/config.yml /etc/metricbeat/metricbeat.yml - -ENTRYPOINT ["metricbeat", "run", "--path.config", "/etc/metricbeat"] \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml similarity index 83% rename from metricbeat/module/pgbouncer/docker-compose.yml rename to metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml index 07caf5f21153..a26c368aa92f 100644 --- a/metricbeat/module/pgbouncer/docker-compose.yml +++ b/metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml @@ -19,13 +19,6 @@ services: volumes: - ./pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini - ./userlist.txt:/etc/pgbouncer/userlist.txt - # metricbeat: - # image: test - # build: - # context: ./../../../../beats - # dockerfile: ./metricbeat/module/pgbouncer/_meta/Dockerfile - # cache_from: - # - test elasticsearch: image: elasticsearch:8.11.0 diff --git a/metricbeat/module/pgbouncer/pgbouncer.ini b/metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini similarity index 100% rename from metricbeat/module/pgbouncer/pgbouncer.ini rename to metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini diff --git a/metricbeat/module/pgbouncer/_meta/compose/userlist.txt b/metricbeat/module/pgbouncer/_meta/compose/userlist.txt new file mode 100644 index 000000000000..1ada9b43f073 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/compose/userlist.txt @@ -0,0 +1 @@ +"test" "password" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config.reference.yml b/metricbeat/module/pgbouncer/_meta/config.reference.yml index e69de29bb2d1..cf62e88df9ef 100644 --- a/metricbeat/module/pgbouncer/_meta/config.reference.yml +++ b/metricbeat/module/pgbouncer/_meta/config.reference.yml @@ -0,0 +1,32 @@ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml index 15e1e260d56c..e48057424bd5 100644 --- a/metricbeat/module/pgbouncer/_meta/config.yml +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -1,19 +1,11 @@ -# Module: pgbouncer -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html - metricbeat.modules: - module: pgbouncer - metricsets: - - stats - - lists - - pools + # metricsets: + # - stats + # - lists + # - pools + # - mem period: 10s - hosts: ["postgresql://pgbouncer:6432/pgbouncer?sslmode=disable"] - username: test - password: password -output.elasticsearch: - hosts: ["http://elasticsearch:9200"] - username: elastic - password: xxxx - indices: - - index: "test" \ No newline at end of file + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + # username: test + # password: password \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index 80562b1ea154..97f5894ace19 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -1,10 +1,7 @@ -# Module: pgbouncer -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html - metricbeat.modules: - module: pgbouncer metricsets: - - lists + - mem period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] username: test diff --git a/metricbeat/module/pgbouncer/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/_meta/docs.asciidoc new file mode 100644 index 000000000000..b01238849d90 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/docs.asciidoc @@ -0,0 +1,51 @@ +include::{libbeat-dir}/shared/integration-link.asciidoc[] + +:modulename!: + +PgBouncer is a popular, lightweight connection pooler for PostgreSQL. This module allows you to monitor PgBouncer using Metricbeat, gathering metrics that help optimize performance and maintain stability. + +This module periodically fetches metrics from the PgBouncer poolers available at https://www.pgbouncer.org/. + +Default metricsets include: +- `lists`: Provides a list of databases and users currently connected to PgBouncer. +- `mem`: Monitors memory usage details within PgBouncer, helping detect memory leaks or spikes. +- `pools`: Tracks the status of connection pools, including active and idle connections. +- `stats`: Collects comprehensive statistics on database connections and queries processed. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use PostgreSQL URLs of the following format with PgBouncer database name included: + +[source,yaml] +----------------------------------- +[postgres://][user:pass@]host[:port]/pgbouncer[?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer:40001?sslmode=disable", "postgres://otherhost/pgbouncer:40001"] +---------------------------------------------------------------------- + +Usernames and passwords specified in the URL take precedence over those specified in the `username` and `password` configuration options. +For security best practices, consider using environment variables to pass sensitive information securely. + +[source,yaml] +---- +- module: postgresql + metricsets: ["pools"] + hosts: ["postgres://localhost/pgbouncer:6432"] + username: root + password: test +---- \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/_meta/data.json b/metricbeat/module/pgbouncer/lists/_meta/data.json new file mode 100644 index 000000000000..49bf14137359 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2024-08-04T16:40:51.299Z", + "event": { + "dataset": "pgbouncer.lists", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "lists", + "period": 10000 + }, + "pgbouncer": { + "lists": { + "databases": 1, + "free_clients": 48, + "free_servers": 0, + "users": 4, + "dns_zones": 0, + "peers": 0, + "peer_pools": 0, + "login_clients": 0, + "pools": 1, + "used_clients": 2, + "used_servers": 0, + "dns_names": 0 + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc new file mode 100644 index 000000000000..c5aa07c744e0 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `lists` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/mem/_meta/data.json b/metricbeat/module/pgbouncer/mem/_meta/data.json new file mode 100644 index 000000000000..43e2d822d952 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/data.json @@ -0,0 +1,86 @@ +{ + "@timestamp": "2024-08-04T17:38:47.013Z", + "event": { + "dataset": "pgbouncer.mem", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "mem", + "period": 10000 + }, + "pgbouncer": { + "mem": { + "credentials_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "peer_pool_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "iobuf_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "pool_cache": { + "used": 1, + "free": 49, + "memtotal": 30800, + "size": 616 + }, + "outstanding_request_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "user_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "server_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "server_prepared_statement_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "db_cache": { + "free": 49, + "memtotal": 30800, + "size": 616, + "used": 1 + }, + "peer_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "var_list_cache": { + "used": 1, + "free": 49, + "memtotal": 30800, + "size": 616 + } + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc new file mode 100644 index 000000000000..bc9bbc13b2a7 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mem` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/mem/_meta/fields.yml b/metricbeat/module/pgbouncer/mem/_meta/fields.yml index 1f383003bed2..cae5423a338c 100644 --- a/metricbeat/module/pgbouncer/mem/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/mem/_meta/fields.yml @@ -1,50 +1,182 @@ -- name: lists +- name: mem type: group description: > - Shows various internal information lists. + Shows cache memory information for various PgBouncer caches. release: ga fields: - - name: databases + - name: user_cache.size type: long description: > - Count of databases. - - name: users + The size of a single slot in the user cache. + - name: user_cache.used type: long description: > - Count of users. - - name: pools + Number of used slots in the user cache. + - name: user_cache.free type: long description: > - Count of pools. - - name: free_clients + Number of available slots in the user cache. + - name: user_cache.memtotal type: long description: > - Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. - - name: used_clients + Total bytes used by the user cache. + - name: credentials_cache.size type: long description: > - Count of used clients. - - name: login_clients + The size of a single slot in the credentials cache. + - name: credentials_cache.used type: long description: > - Count of clients in login state. - - name: free_servers + Number of used slots in the credentials cache. + - name: credentials_cache.free type: long description: > - Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. - - name: used_servers + Number of available slots in the credentials cache. + - name: credentials_cache.memtotal type: long description: > - Count of used servers. - - name: dns_names + Total bytes used by the credentials cache. + - name: db_cache.size type: long description: > - Count of DNS names in the cache. - - name: dns_zones + The size of a single slot in the db cache. + - name: db_cache.used type: long description: > - Count of DNS zones in the cache. - - name: dns_queries + Number of used slots in the db cache. + - name: db_cache.free type: long description: > - Count of in-flight DNS queries. \ No newline at end of file + Number of available slots in the db cache. + - name: db_cache.memtotal + type: long + description: > + Total bytes used by the db cache. + - name: peer_cache.size + type: long + description: > + The size of a single slot in the peer cache. + - name: peer_cache.used + type: long + description: > + Number of used slots in the peer cache. + - name: peer_cache.free + type: long + description: > + Number of available slots in the peer cache. + - name: peer_cache.memtotal + type: long + description: > + Total bytes used by the peer cache. + - name: peer_pool_cache.size + type: long + description: > + The size of a single slot in the peer pool cache. + - name: peer_pool_cache.used + type: long + description: > + Number of used slots in the peer pool cache. + - name: peer_pool_cache.free + type: long + description: > + Number of available slots in the peer pool cache. + - name: peer_pool_cache.memtotal + type: long + description: > + Total bytes used by the peer pool cache. + - name: pool_cache.size + type: long + description: > + The size of a single slot in the pool cache. + - name: pool_cache.used + type: long + description: > + Number of used slots in the pool cache. + - name: pool_cache.free + type: long + description: > + Number of available slots in the pool cache. + - name: pool_cache.memtotal + type: long + description: > + Total bytes used by the pool cache. + - name: outstanding_request_cache.size + type: long + description: > + The size of a single slot in the outstanding request cache. + - name: outstanding_request_cache.used + type: long + description: > + Number of used slots in the outstanding request cache. + - name: outstanding_request_cache.free + type: long + description: > + Number of available slots in the outstanding request cache. + - name: outstanding_request_cache.memtotal + type: long + description: > + Total bytes used by the outstanding request cache. + - name: server_cache.size + type: long + description: > + The size of a single slot in the server cache. + - name: server_cache.used + type: long + description: > + Number of used slots in the server cache. + - name: server_cache.free + type: long + description: > + Number of available slots in the server cache. + - name: server_cache.memtotal + type: long + description: > + Total bytes used by the server cache. + - name: iobuf_cache.size + type: long + description: > + The size of a single slot in the iobuf cache. + - name: iobuf_cache.used + type: long + description: > + Number of used slots in the iobuf cache. + - name: iobuf_cache.free + type: long + description: > + Number of available slots in the iobuf cache. + - name: iobuf_cache.memtotal + type: long + description: > + Total bytes used by the iobuf cache. + - name: var_list_cache.size + type: long + description: > + The size of a single slot in the var list cache. + - name: var_list_cache.used + type: long + description: > + Number of used slots in the var list cache. + - name: var_list_cache.free + type: long + description: > + Number of available slots in the var list cache. + - name: var_list_cache.memtotal + type: long + description: > + Total bytes used by the var list cache. + - name: server_prepared_statement_cache.size + type: long + description: > + The size of a single slot in the server prepared statement cache. + - name: server_prepared_statement_cache.used + type: long + description: > + Number of used slots in the server prepared statement cache. + - name: server_prepared_statement_cache.free + type: long + description: > + Number of available slots in the server prepared statement cache. + - name: server_prepared_statement_cache.memtotal + type: long + description: > + Total bytes used by the server prepared statement cache. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/_meta/data.json b/metricbeat/module/pgbouncer/pools/_meta/data.json new file mode 100644 index 000000000000..42521a9badca --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2024-08-04T16:48:49.028Z", + "event": { + "dataset": "pgbouncer.pools", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "pools", + "period": 10000 + }, + "pgbouncer": { + "pools": { + "sv_tested": 0, + "cl_waiting": 0, + "sv_login": 0, + "maxwait_us": 0, + "pool_mode": "statement", + "cl_active": 3, + "database": "pgbouncer", + "sv_active_cancel": 0, + "user": "pgbouncer", + "cl_active_cancel_req": 0, + "sv_being_canceled": 0, + "sv_idle": 0, + "cl_waiting_cancel_req": 0, + "sv_active": 0, + "sv_used": 0 + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc new file mode 100644 index 000000000000..3bf80cfc02eb --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `pools` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/stats/_meta/data.json b/metricbeat/module/pgbouncer/stats/_meta/data.json new file mode 100644 index 000000000000..561dddc866ed --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/data.json @@ -0,0 +1,37 @@ +{ + "@timestamp": "2024-08-04T16:49:49.268Z", + "event": { + "dataset": "pgbouncer.stats", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "stats", + "period": 10000 + }, + "pgbouncer": { + "stats": { + "total_server_assignment_count": 0, + "total_sent": 0, + "avg_sent": 0, + "total_wait_time_us": 0, + "database": "pgbouncer", + "total_xact_count": 25, + "total_received": 0, + "total_query_count": 25, + "avg_query_count": 0, + "avg_xact_time_us": 0, + "avg_query_time_us": 0, + "total_query_time_us": 0, + "avg_recv": 0, + "total_xact_time_us": 0, + "avg_xact_count": 0, + "avg_wait_time_us": 0, + "avg_server_assignment_count": 0 + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc new file mode 100644 index 000000000000..713f5d3f61e7 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `stats` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go index 92a2967a45a0..4a720a024da7 100644 --- a/metricbeat/module/pgbouncer/stats/data.go +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -29,16 +29,16 @@ var schema = s.Schema{ "total_server_assignment_count": c.Int("total_server_assignment_count"), "total_received": c.Int("total_received"), "total_sent": c.Int("total_sent"), - "total_xact_time": c.Int("total_xact_time"), - "total_query_time": c.Int("total_query_time"), - "total_wait_time": c.Int("total_wait_time"), + "total_xact_time_us": c.Int("total_xact_time"), + "total_query_time_us": c.Int("total_query_time"), + "total_wait_time_us": c.Int("total_wait_time"), "total_xact_count": c.Int("total_xact_count"), "avg_xact_count": c.Int("avg_xact_count"), "avg_query_count": c.Int("avg_query_count"), "avg_server_assignment_count": c.Int("avg_server_assignment_count"), "avg_recv": c.Int("avg_recv"), "avg_sent": c.Int("avg_sent"), - "avg_xact_time": c.Int("avg_xact_time"), - "avg_query_time": c.Int("avg_query_time"), - "avg_wait_time": c.Int("avg_wait_time"), + "avg_xact_time_us": c.Int("avg_xact_time"), + "avg_query_time_us": c.Int("avg_query_time"), + "avg_wait_time_us": c.Int("avg_wait_time"), } diff --git a/metricbeat/module/pgbouncer/todo b/metricbeat/module/pgbouncer/todo deleted file mode 100644 index e9c0616cebb6..000000000000 --- a/metricbeat/module/pgbouncer/todo +++ /dev/null @@ -1,10 +0,0 @@ - - -ogarnąć typy (ms, s itd) - -dostosowac konfiguracje metricbeat dla pgbouncer - -usunac jakos connection string z logow - -napisac testy - diff --git a/metricbeat/module/pgbouncer/userlist.txt b/metricbeat/module/pgbouncer/userlist.txt deleted file mode 100644 index a8545ab8caa8..000000000000 --- a/metricbeat/module/pgbouncer/userlist.txt +++ /dev/null @@ -1,3 +0,0 @@ -"test" "password" -"user1" "password" -"user2" "password" \ No newline at end of file From a7d301a1cba0323a33a1888923fa5f735c81b423 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:31 +0200 Subject: [PATCH 12/53] add tests --- go.mod | 1 + go.sum | 2 + .../module/pgbouncer/_meta/pgbouncer.ini | 18 ++ .../module/pgbouncer/_meta/userlist.txt | 1 + .../module/pgbouncer/docker-compose.yml | 46 +++++ .../pgbouncer/lists/lists_integration_test.go | 42 +++++ .../pgbouncer/mem/mem_integration_test.go | 42 +++++ metricbeat/module/pgbouncer/metricset_test.go | 97 ++++++++++ metricbeat/module/pgbouncer/pgbouncer_test.go | 166 ++++++++++++++++++ .../pgbouncer/pools/pools_integration_test.go | 40 +++++ .../pgbouncer/stats/stats_integration_test.go | 40 +++++ 11 files changed, 495 insertions(+) create mode 100644 metricbeat/module/pgbouncer/_meta/pgbouncer.ini create mode 100644 metricbeat/module/pgbouncer/_meta/userlist.txt create mode 100644 metricbeat/module/pgbouncer/docker-compose.yml create mode 100644 metricbeat/module/pgbouncer/lists/lists_integration_test.go create mode 100644 metricbeat/module/pgbouncer/mem/mem_integration_test.go create mode 100644 metricbeat/module/pgbouncer/metricset_test.go create mode 100644 metricbeat/module/pgbouncer/pgbouncer_test.go create mode 100644 metricbeat/module/pgbouncer/pools/pools_integration_test.go create mode 100644 metricbeat/module/pgbouncer/stats/stats_integration_test.go diff --git a/go.mod b/go.mod index 55ef8b4acc5a..87812b61b57a 100644 --- a/go.mod +++ b/go.mod @@ -366,6 +366,7 @@ require ( go.opentelemetry.io/otel/trace v1.19.0 // indirect golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect golang.org/x/term v0.13.0 // indirect + golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect diff --git a/go.sum b/go.sum index df5741e581b7..bc666d261374 100644 --- a/go.sum +++ b/go.sum @@ -2475,6 +2475,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools/cmd/cover v0.1.0-deprecated h1:Rwy+mWYz6loAF+LnG1jHG/JWMHRMMC2/1XX3Ejkx9lA= +golang.org/x/tools/cmd/cover v0.1.0-deprecated/go.mod h1:hMDiIvlpN1NoVgmjLjUJE9tMHyxHjFX7RuQ+rW12mSA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/metricbeat/module/pgbouncer/_meta/pgbouncer.ini b/metricbeat/module/pgbouncer/_meta/pgbouncer.ini new file mode 100644 index 000000000000..e187236e43d1 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/pgbouncer.ini @@ -0,0 +1,18 @@ +[databases] +* = host=postgresql port=5432 dbname=test + +[users] +test = pool_mode=statement max_user_connections=40 + +[pgbouncer] +listen_addr = 0.0.0.0 +listen_port = 6432 +auth_type = trust +admin_users = test +auth_file = /etc/pgbouncer/userlist.txt +max_client_conn = 8000 +default_pool_size = 400 +server_idle_timeout = 120 + +ignore_startup_parameters = extra_float_digits +log_connections = 0 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/userlist.txt b/metricbeat/module/pgbouncer/_meta/userlist.txt new file mode 100644 index 000000000000..1ada9b43f073 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/userlist.txt @@ -0,0 +1 @@ +"test" "password" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml new file mode 100644 index 000000000000..76ce86c9d90e --- /dev/null +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.9' + +services: + postgresql: + image: postgres:15 + hostname: postgresql + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_USER=test + - POSTGRES_DB=test + - POSTGRES_HOST_AUTH_METHOD=md5 + - POSTGRES_INITDB_ARGS=--auth=md5 + + pgbouncer: + image: edoburu/pgbouncer + hostname: pgbouncer + ports: + - 6432:6432 + volumes: + - ./_meta/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini + - ./_meta/userlist.txt:/etc/pgbouncer/userlist.txt + healthcheck: + test: ['CMD-SHELL', 'psql -h localhost -p 6432 -U test -d pgbouncer -c "SHOW STATS"'] + interval: 15s + timeout: 30s + retries: 5 + start_period: 15s + + elasticsearch: + image: elasticsearch:8.11.0 + hostname: elasticsearch + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "http.host=0.0.0.0" + - "xpack.security.enabled=false" + - "discovery.type=single-node" + # - "transport.host=127.0.0.1" + ports: + - 9200:9200 + + kibana: + image: kibana:8.11.0 + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + ports: + - 5601:5601 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go new file mode 100644 index 000000000000..396470c17cee --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -0,0 +1,42 @@ +package lists + +import ( + "fmt" + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + fmt.Printf("%v", f) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + assert.Contains(t, event, "databases") + assert.Contains(t, event, "users") + assert.Contains(t, event, "peers") + assert.Contains(t, event, "pools") + assert.Contains(t, event, "peer_pools") + assert.Contains(t, event, "used_clients") + assert.Contains(t, event, "free_servers") + assert.Contains(t, event, "used_servers") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"lists"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go new file mode 100644 index 000000000000..3e5220058bd2 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -0,0 +1,42 @@ +package mem + +import ( + "fmt" + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + fmt.Printf("%v", f) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + assert.Contains(t, event["user_cache"], "size") + assert.Contains(t, event["user_cache"], "used") + assert.Contains(t, event["user_cache"], "free") + assert.Contains(t, event["user_cache"], "memtotal") + assert.Contains(t, event["credentials_cache"], "size") + assert.Contains(t, event["db_cache"], "used") + assert.Contains(t, event["peer_cache"], "free") + assert.Contains(t, event["iobuf_cache"], "memtotal") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"mem"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/metricset_test.go b/metricbeat/module/pgbouncer/metricset_test.go new file mode 100644 index 000000000000..af7bbad4bcd7 --- /dev/null +++ b/metricbeat/module/pgbouncer/metricset_test.go @@ -0,0 +1,97 @@ +package pgbouncer + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/stretchr/testify/assert" +) + +func TestNewMetricSet(t *testing.T) { + base := mb.BaseMetricSet{} + metricSet, err := NewMetricSet(base) + assert.NoError(t, err) + assert.NotNil(t, metricSet) +} + +func TestDBConnection(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + defer db.Close() + ctx := context.Background() + metricSet := MetricSet{ + db: db, + } + conn, err := metricSet.DB(ctx) + assert.NoError(t, err) + assert.NotNil(t, conn, "The database connection should not be nil") + if conn != nil { + defer conn.Close() + } +} + +func TestQueryStats(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + + defer db.Close() + metricSet := MetricSet{ + db: db, + } + ctx := context.Background() + query := "SHOW STATS;" + results, err := metricSet.QueryStats(ctx, query) + assert.NoError(t, err) + assert.NotNil(t, results) + assert.NotEmpty(t, results) +} + +func TestClose(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + + metricSet := MetricSet{ + db: db, + } + + err = metricSet.Close() + assert.NoError(t, err) + + err = db.Ping() + assert.Error(t, err) +} + +func connectDatabase(t *testing.T) (*sql.DB, error) { + service := compose.EnsureUp(t, "pgbouncer") + config := getConfig(service.Host()) + + dsn := fmt.Sprintf("postgres://%s:%s@%s", + config["username"].(string), + config["password"].(string), + config["hosts"].([]string)[0], + ) + + db, err := sql.Open("postgres", dsn) + return db, err +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"stats"}, + "hosts": []string{fmt.Sprintf("localhost:6432/pgbouncer?sslmode=disable")}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go new file mode 100644 index 000000000000..d9e749fec60d --- /dev/null +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -0,0 +1,166 @@ +package pgbouncer + +import ( + "fmt" + "testing" + "time" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/stretchr/testify/assert" +) + +func TestParseUrl(t *testing.T) { + tests := []struct { + Name string + URL string + Username string + Password string + Timeout time.Duration + Expected string + ExpectErr bool + RequireUsername bool + }{ + { + Name: "simple test", + URL: "postgres://host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "no port", + URL: "postgres://host1/pgbouncer", + Expected: "dbname='pgbouncer' host='host1'", + ExpectErr: false, + }, + { + Name: "user/pass in URL", + URL: "postgres://user:pass@host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' password='pass' port='6432' user='user'", + ExpectErr: false, + }, + { + Name: "user/pass in params", + URL: "postgres://host1:6432/pgbouncer", + Username: "user", + Password: "secret", + Expected: "dbname='pgbouncer' host='host1' password='secret' port='6432' user='user'", + ExpectErr: false, + }, + { + Name: "user/pass in URL take precedence", + URL: "postgres://user1:pass@host1:6432/pgbouncer", + Username: "user", + Password: "secret", + Expected: "dbname='pgbouncer' host='host1' password='pass' port='6432' user='user1'", + ExpectErr: false, + }, + { + Name: "timeout no override", + URL: "postgres://host1:6432/pgbouncer?connect_timeout=2", + Expected: "connect_timeout='2' dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "timeout from param", + URL: "postgres://host1:6432/pgbouncer", + Timeout: 3 * time.Second, + Expected: "connect_timeout='3' dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "user/pass in URL take precedence, and timeout override", + URL: "postgres://user1:pass@host1:6432/pgbouncer?connect_timeout=2", + Username: "user", + Password: "secret", + Timeout: 3 * time.Second, + Expected: "connect_timeout='3' dbname='pgbouncer' host='host1' password='pass' port='6432' user='user1'", + ExpectErr: false, + }, + { + Name: "unix socket", + URL: "postgresql:///pgbouncer?host=/var/lib/postgresql", + Expected: "dbname='pgbouncer' host='/var/lib/postgresql'", + ExpectErr: false, + }, + { + Name: "no ssl", + URL: "postgresql://localhost:6432/pgbouncer?sslmode=disable", + Expected: "dbname='pgbouncer' host='localhost' port='6432' sslmode='disable'", + ExpectErr: false, + }, + { + Name: "no scheme", + URL: "host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "invalid url", + URL: "://pgbouncer:6432", + ExpectErr: true, + }, + { + Name: "empty username", + URL: "postgres://localhost:5432", + RequireUsername: true, + Password: "pass", + ExpectErr: true, + }, + { + Name: "invalid schema", + URL: "abcd://", + ExpectErr: true, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + mod := &MockModule{ + Username: test.Username, + Password: test.Password, + Timeout: test.Timeout, + RequireUsername: test.RequireUsername, // Ustawienie na podstawie testu + } + + hostData, err := ParseURL(mod, test.URL) + + if test.ExpectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.Expected, hostData.URI) + } + }) + } + +} + +type MockModule struct { + Username string + Password string + Timeout time.Duration + RequireUsername bool // Defaultowo false +} + +func (m *MockModule) UnpackConfig(to interface{}) error { + if m.RequireUsername && m.Username == "" { + return fmt.Errorf("no username provided") + } + c := to.(*struct { + Username string `config:"username"` + Password string `config:"password"` + }) + c.Username = m.Username + c.Password = m.Password + return nil +} + +func (m *MockModule) Config() mb.ModuleConfig { + return mb.ModuleConfig{ + Timeout: m.Timeout, + } +} + +func (m *MockModule) Name() string { + return "mockmodule" +} diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go new file mode 100644 index 000000000000..152761b73b79 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -0,0 +1,40 @@ +package pools + +import ( + "fmt" + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + fmt.Printf("%v", f) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + assert.Contains(t, event, "user") + assert.Contains(t, event, "cl_active") + assert.Contains(t, event, "cl_active_cancel_req") + assert.Contains(t, event, "sv_idle") + assert.Contains(t, event, "maxwait_us") + assert.Contains(t, event, "pool_mode") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"pools"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go new file mode 100644 index 000000000000..c0739a69da30 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -0,0 +1,40 @@ +package stats + +import ( + "fmt" + "testing" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + fmt.Printf("%v", f) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + assert.Contains(t, event, "total_xact_count") + assert.Contains(t, event, "total_server_assignment_count") + assert.Contains(t, event, "total_wait_time_us") + assert.Contains(t, event, "database") + assert.Contains(t, event, "total_received") + assert.Contains(t, event, "total_query_count") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"stats"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} From a929b0b6657bf16383504881996b48883b91ea4a Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:32 +0200 Subject: [PATCH 13/53] cleanup --- .../_meta/compose/docker-compose.yml | 40 ------------------- .../pgbouncer/_meta/compose/pgbouncer.ini | 21 ---------- .../pgbouncer/_meta/compose/userlist.txt | 1 - .../module/pgbouncer/_meta/config_local.yml | 3 ++ 4 files changed, 3 insertions(+), 62 deletions(-) delete mode 100644 metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml delete mode 100644 metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini delete mode 100644 metricbeat/module/pgbouncer/_meta/compose/userlist.txt diff --git a/metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml b/metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml deleted file mode 100644 index a26c368aa92f..000000000000 --- a/metricbeat/module/pgbouncer/_meta/compose/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: '3.9' - -services: - postgresql: - image: postgres:15 - hostname: postgresql - environment: - - POSTGRES_PASSWORD=password - - POSTGRES_USER=test - - POSTGRES_DB=test - - POSTGRES_HOST_AUTH_METHOD=md5 - - POSTGRES_INITDB_ARGS=--auth=md5 - - pgbouncer: - image: edoburu/pgbouncer - hostname: pgbouncer - ports: - - 6432:6432 - volumes: - - ./pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini - - ./userlist.txt:/etc/pgbouncer/userlist.txt - - elasticsearch: - image: elasticsearch:8.11.0 - hostname: elasticsearch - environment: - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "http.host=0.0.0.0" - - "xpack.security.enabled=false" - - "discovery.type=single-node" - # - "transport.host=127.0.0.1" - ports: - - 9200:9200 - - kibana: - image: kibana:8.11.0 - environment: - - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 - ports: - - 5601:5601 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini b/metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini deleted file mode 100644 index c8826244eb43..000000000000 --- a/metricbeat/module/pgbouncer/_meta/compose/pgbouncer.ini +++ /dev/null @@ -1,21 +0,0 @@ -[databases] -* = host=postgresql port=5432 dbname=test - -[users] -# dostosować wartości max_user_connections do wielkości bazy -test = pool_mode=statement max_user_connections=40 -user1 = pool_mode=transaction max_user_connections=25 -user2 = pool_mode=transaction max_user_connections=25 - -[pgbouncer] -listen_addr = 0.0.0.0 -listen_port = 6432 -auth_type = trust -admin_users = test -auth_file = /etc/pgbouncer/userlist.txt -max_client_conn = 8000 -default_pool_size = 400 -server_idle_timeout = 120 - -ignore_startup_parameters = extra_float_digits -log_connections = 0 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/compose/userlist.txt b/metricbeat/module/pgbouncer/_meta/compose/userlist.txt deleted file mode 100644 index 1ada9b43f073..000000000000 --- a/metricbeat/module/pgbouncer/_meta/compose/userlist.txt +++ /dev/null @@ -1 +0,0 @@ -"test" "password" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml index 97f5894ace19..a69d02881b67 100644 --- a/metricbeat/module/pgbouncer/_meta/config_local.yml +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -2,6 +2,9 @@ metricbeat.modules: - module: pgbouncer metricsets: - mem + - lists + - stats + - pools period: 10s hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] username: test From 378fcdac74e133a5a3cd25409f8b809a1f21f7ef Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:25:32 +0200 Subject: [PATCH 14/53] formatting --- metricbeat/module/pgbouncer/doc.go | 2 +- metricbeat/module/pgbouncer/metricset.go | 8 +++++++- metricbeat/module/pgbouncer/pgbouncer_test.go | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/pgbouncer/doc.go b/metricbeat/module/pgbouncer/doc.go index c8dc6850145e..0f2098d2e074 100644 --- a/metricbeat/module/pgbouncer/doc.go +++ b/metricbeat/module/pgbouncer/doc.go @@ -18,4 +18,4 @@ /* Package pgbouncer is a Metricbeat module that contains MetricSets. */ -package pgbouncer \ No newline at end of file +package pgbouncer diff --git a/metricbeat/module/pgbouncer/metricset.go b/metricbeat/module/pgbouncer/metricset.go index 2083111e7d7e..0a16ef8e2025 100644 --- a/metricbeat/module/pgbouncer/metricset.go +++ b/metricbeat/module/pgbouncer/metricset.go @@ -1,4 +1,5 @@ package pgbouncer + import ( "context" "database/sql" @@ -8,14 +9,17 @@ import ( // Register pgbouncer database/sql driver _ "github.com/lib/pq" ) + type MetricSet struct { mb.BaseMetricSet db *sql.DB } + // NewMetricSet creates a pgbouncer metricset with a pool of connections func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return &MetricSet{BaseMetricSet: base}, nil } + // DB creates a database connection, it must be freed after use with `Close()` func (ms *MetricSet) DB(ctx context.Context) (*sql.Conn, error) { if ms.db == nil { @@ -27,6 +31,7 @@ func (ms *MetricSet) DB(ctx context.Context) (*sql.Conn, error) { } return ms.db.Conn(ctx) } + // QueryStats makes the database call for a given metric func (ms *MetricSet) QueryStats(ctx context.Context, query string) ([]map[string]interface{}, error) { db, err := ms.DB(ctx) @@ -62,6 +67,7 @@ func (ms *MetricSet) QueryStats(ctx context.Context, query string) ([]map[string } return results, nil } + // Close closes the metricset and its connections func (ms *MetricSet) Close() error { if ms.db == nil { @@ -71,4 +77,4 @@ func (ms *MetricSet) Close() error { return fmt.Errorf("failed to close connection: %w", err) } return nil -} \ No newline at end of file +} diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go index d9e749fec60d..16cd62eba1ad 100644 --- a/metricbeat/module/pgbouncer/pgbouncer_test.go +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -119,7 +119,7 @@ func TestParseUrl(t *testing.T) { Username: test.Username, Password: test.Password, Timeout: test.Timeout, - RequireUsername: test.RequireUsername, // Ustawienie na podstawie testu + RequireUsername: test.RequireUsername, } hostData, err := ParseURL(mod, test.URL) @@ -139,7 +139,7 @@ type MockModule struct { Username string Password string Timeout time.Duration - RequireUsername bool // Defaultowo false + RequireUsername bool } func (m *MockModule) UnpackConfig(to interface{}) error { From 4f0ea6974ae6203be75cf5bfa14c85370455e915 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:52:35 +0200 Subject: [PATCH 15/53] Add the license header --- metricbeat/module/pgbouncer/lists/data.go | 17 +++++++++++++++++ metricbeat/module/pgbouncer/lists/lists.go | 17 +++++++++++++++++ .../pgbouncer/lists/lists_integration_test.go | 17 +++++++++++++++++ metricbeat/module/pgbouncer/mem/data.go | 17 +++++++++++++++++ metricbeat/module/pgbouncer/mem/mem.go | 17 +++++++++++++++++ .../pgbouncer/mem/mem_integration_test.go | 17 +++++++++++++++++ metricbeat/module/pgbouncer/metricset.go | 19 +++++++++++++++++++ metricbeat/module/pgbouncer/pgbouncer_test.go | 17 +++++++++++++++++ .../pgbouncer/pools/pools_integration_test.go | 17 +++++++++++++++++ .../pgbouncer/stats/stats_integration_test.go | 17 +++++++++++++++++ 10 files changed, 172 insertions(+) diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 3ea03c6944e9..111871cc6174 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package lists import ( diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 618192409fca..9b81efe33068 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package lists import ( diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index 396470c17cee..8d91fc6cf616 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package lists import ( diff --git a/metricbeat/module/pgbouncer/mem/data.go b/metricbeat/module/pgbouncer/mem/data.go index 7bd54733c207..a719893ac313 100644 --- a/metricbeat/module/pgbouncer/mem/data.go +++ b/metricbeat/module/pgbouncer/mem/data.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package mem import ( diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index 94cd9d39eef5..a6c11748bed4 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package mem import ( diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go index 3e5220058bd2..497838f4e342 100644 --- a/metricbeat/module/pgbouncer/mem/mem_integration_test.go +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package mem import ( diff --git a/metricbeat/module/pgbouncer/metricset.go b/metricbeat/module/pgbouncer/metricset.go index 0a16ef8e2025..cf9f179d98e0 100644 --- a/metricbeat/module/pgbouncer/metricset.go +++ b/metricbeat/module/pgbouncer/metricset.go @@ -1,11 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package pgbouncer import ( "context" "database/sql" "fmt" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" + // Register pgbouncer database/sql driver _ "github.com/lib/pq" ) diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go index 16cd62eba1ad..19b1aa042625 100644 --- a/metricbeat/module/pgbouncer/pgbouncer_test.go +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package pgbouncer import ( diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index 152761b73b79..288a78fbed74 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package pools import ( diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index c0739a69da30..4668f22a1c15 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package stats import ( From 869567822643252d52b47d1eabe668850b3bb678 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 16:56:50 +0200 Subject: [PATCH 16/53] Add the license header --- metricbeat/module/pgbouncer/metricset_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metricbeat/module/pgbouncer/metricset_test.go b/metricbeat/module/pgbouncer/metricset_test.go index af7bbad4bcd7..55047d162096 100644 --- a/metricbeat/module/pgbouncer/metricset_test.go +++ b/metricbeat/module/pgbouncer/metricset_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package pgbouncer import ( From be437de32a4558057b91643098a84db003435310 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 17:42:45 +0200 Subject: [PATCH 17/53] Refactor ParseURL tests with MockModule --- metricbeat/module/pgbouncer/pgbouncer_test.go | 86 +++++++++++-------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go index 19b1aa042625..4184c51191da 100644 --- a/metricbeat/module/pgbouncer/pgbouncer_test.go +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -22,20 +22,22 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/management/status" "github.com/elastic/beats/v7/metricbeat/mb" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" ) func TestParseUrl(t *testing.T) { tests := []struct { - Name string - URL string - Username string - Password string - Timeout time.Duration - Expected string - ExpectErr bool - RequireUsername bool + Name string + URL string + Username string + Password string + Timeout time.Duration + Expected string + ExpectErr bool }{ { Name: "simple test", @@ -116,28 +118,19 @@ func TestParseUrl(t *testing.T) { URL: "://pgbouncer:6432", ExpectErr: true, }, - { - Name: "empty username", - URL: "postgres://localhost:5432", - RequireUsername: true, - Password: "pass", - ExpectErr: true, - }, { Name: "invalid schema", URL: "abcd://", ExpectErr: true, }, } - for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - mod := &MockModule{ - Username: test.Username, - Password: test.Password, - Timeout: test.Timeout, - RequireUsername: test.RequireUsername, - } + mod := mbtest.NewTestModule(t, map[string]interface{}{ + "username": test.Username, + "password": test.Password, + }) + mod.ModConfig.Timeout = test.Timeout hostData, err := ParseURL(mod, test.URL) @@ -149,19 +142,38 @@ func TestParseUrl(t *testing.T) { } }) } - } type MockModule struct { - Username string - Password string - Timeout time.Duration - RequireUsername bool + Username string + Password string + Timeout time.Duration + statusReporter status.StatusReporter + statusUpdates []statusUpdate +} +type statusUpdate struct { + status status.Status + msg string +} + +func (m *MockModule) Name() string { + return "mockmodule" +} +func (m *MockModule) SetStatusReporter(statusReporter status.StatusReporter) { + m.statusReporter = statusReporter +} +func (m *MockModule) UpdateStatus(status status.Status, msg string) { + m.statusUpdates = append(m.statusUpdates, statusUpdate{status: status, msg: msg}) +} +func (m *MockModule) Config() mb.ModuleConfig { + return mb.ModuleConfig{ + Timeout: m.Timeout, + } } func (m *MockModule) UnpackConfig(to interface{}) error { - if m.RequireUsername && m.Username == "" { - return fmt.Errorf("no username provided") + if m.Username == "" { + return fmt.Errorf("simulated config error") } c := to.(*struct { Username string `config:"username"` @@ -171,13 +183,13 @@ func (m *MockModule) UnpackConfig(to interface{}) error { c.Password = m.Password return nil } - -func (m *MockModule) Config() mb.ModuleConfig { - return mb.ModuleConfig{ - Timeout: m.Timeout, +func TestParseURL_UnpackConfigError(t *testing.T) { + mod := &MockModule{ + Username: "", + Password: "pass", + Timeout: 5 * time.Second, } -} - -func (m *MockModule) Name() string { - return "mockmodule" + _, err := ParseURL(mod, "postgres://localhost:5432") + assert.Error(t, err) + assert.Contains(t, err.Error(), "simulated config error") } From f6523bb08b2f80427c67bf299135219232065586 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 17:51:38 +0200 Subject: [PATCH 18/53] Cleanup unnecessary print --- metricbeat/module/pgbouncer/lists/lists_integration_test.go | 2 -- metricbeat/module/pgbouncer/pools/pools_integration_test.go | 2 -- metricbeat/module/pgbouncer/stats/stats_integration_test.go | 2 -- 3 files changed, 6 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index 8d91fc6cf616..2020784d812f 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -18,7 +18,6 @@ package lists import ( - "fmt" "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -32,7 +31,6 @@ func TestMetricSet_Fetch(t *testing.T) { service := compose.EnsureUp(t, "pgbouncer") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - fmt.Printf("%v", f) events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index 288a78fbed74..8a454090c301 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -18,7 +18,6 @@ package pools import ( - "fmt" "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -32,7 +31,6 @@ func TestMetricSet_Fetch(t *testing.T) { service := compose.EnsureUp(t, "pgbouncer") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - fmt.Printf("%v", f) events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index 4668f22a1c15..112d9193c67a 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -18,7 +18,6 @@ package stats import ( - "fmt" "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -32,7 +31,6 @@ func TestMetricSet_Fetch(t *testing.T) { service := compose.EnsureUp(t, "pgbouncer") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - fmt.Printf("%v", f) events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) From 3030408371c3161f11b3575be8b1c4204ad6aabb Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 17:54:38 +0200 Subject: [PATCH 19/53] Add safe type assertion in Fetch --- metricbeat/module/pgbouncer/lists/lists.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 9b81efe33068..71559303c196 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -61,10 +61,12 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } resultMap := make(map[string]interface{}) for _, s := range results { - key := s["list"].(string) + listValue, ok := s["list"].(string) + if !ok { + return fmt.Errorf("expected string type for 'list' but got something else") + } + key := listValue value := s["items"] - - // Assign the value from "items" resultMap[key] = value } data, err := schema.Apply(resultMap) @@ -75,6 +77,5 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { reporter.Event(mb.Event{ MetricSetFields: data, }) - return nil } From 58271ac4008d365cb06fc26e5a01ce0230cad79e Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:22:19 +0200 Subject: [PATCH 20/53] Cleanup unnecessary print --- metricbeat/module/pgbouncer/mem/mem_integration_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go index 497838f4e342..2779eb4da95c 100644 --- a/metricbeat/module/pgbouncer/mem/mem_integration_test.go +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -18,7 +18,6 @@ package mem import ( - "fmt" "testing" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -32,7 +31,6 @@ func TestMetricSet_Fetch(t *testing.T) { service := compose.EnsureUp(t, "pgbouncer") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - fmt.Printf("%v", f) events, errs := mbtest.ReportingFetchV2Error(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) From 56a21a00305c76d665180597a1e16873695b2f75 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:23:44 +0200 Subject: [PATCH 21/53] Update --- metricbeat/module/pgbouncer/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go index d7d5175a42f7..ebc6b711fce0 100644 --- a/metricbeat/module/pgbouncer/fields.go +++ b/metricbeat/module/pgbouncer/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPgbouncer returns asset data. // This is the base64 encoded zlib format compressed contents of module/pgbouncer. func AssetPgbouncer() string { - return "eJzsWs+O2zYTv+9TDHJKgI0fYA8f8LUFemmCtJuehRE1lomlOA5J2as+fTEkJcteebPeSpsWsE8BSc385u9vyOxHeKDuDrZ1ya1V5G4Agg6G7uDdl19/SmvvbgAq8srpbdBs7+B/NwAAnyg4rTwoNoZUoArWjpuDqNUNgN+wC4Viu9b1HazReLoBcGQIPd1BjTcAa02m8ndR5kew2NAxHvmFbivHHbfbvDIBSH4DaGgSvFXeGmsZazLaBz+sTml6Rpv87je897BDp7n1oG0gZ9GAtmt2DcoXScdq9NWxA/rfKcQxzAoDlujJH+32cA3b+mTjGcTy+5lbG4DXB8GrSb2tJze7zih0Wt+W2cyuLwqd1rd2RIUymmyYXa3Ihix7BV835AnQDUsQNhjiQqW9YmtjEd1C2Qb4Uvdp/EC0laMEDTXsOkDHra3Sx3v0gMawwlh+7ORgA55BB1BooSRw1Pq8uW5DO9bPgDvWVS9Csz2fBdVSXoroei9Najdca7uU+t4Z2iY94AMGeiZZPLndAjURkyXLHidLXvphyTLovyxZFvJSRNd7abpRWl/Iv2ZX/cvn+6gipoq4WKHanEkUAfEX22VARMEvBPGtJafnh6Htx7XR9SZEQFnJ6sqtJ266cuuVW6/ceuXWK7deuXVubj1t5q/k1mhn6xzZkKpTIMTFVAdCsU9a/OspdtJZD9Tt2VWX+eszNgPWXvp5ml1Gr0ie1qlMgSro3bTBr8iO2ElHQRk1LdJhQw6Mtg9USS9JNXx0mF08qitDsNdhA5b71II96qBtLV+WBFvHiryn6qxh+fzClm1wJw3ShgGntOO4ajlARwFqDoBPrf1ORAqFVpEpHH17CxPW7PboKqqiHR0k7QbzOY6JlG1AW8Uw9RHJPNJvO/JbtufS/BCZN7ZQwnGJlR2FaQv8bt6iuX9aB0PRHKoF85jyHUzZqW8ALbdj0/VelVTA7FJw9K0lfx5tSYcMoNPe9s/gZoxW7l7GdKC4NRWUpLjJrUVKdJzAgaFimYJaG7SR8WZEb8cG+ZRMiputIRmi0lxFLneBwP0HmHOMhaW1z2l11iEC7A2i1to4P0kJ66ahSmMg00HrsTSxC+Q0e8roJ3hFzMJ4o6dLIpuiJuAadiSbNruzUBtSD0VFBrtbiWDYUAeWhhExH0ihCAyutSki1EBJ6ySOun74Tc6pUZ9pz35XBPJhcdOPC8y11kqeZgrNljnyFLJl7CbsPWtCvOUsZ8EBeJ5QM1nLPGK4rsWUcw5u8FGqsmjnGlg/aeXYk2JbwRZd6IeiBh910zaHHqAbWsEftBW3pqs5QeCAJh6J+2JPc5D3zPtF0XA14xT5VZzIbASoSBYg7Zhhh+AGnOMpS8RoH7Ty4LVVdBRFH9CF/8KcnfpuieqBpN15GC7NEHg6djHgqXoKJVedmbLwa0wk2zYlOYF5//tvwiEN2srHyFIFZXf8n5Ln0OVCR+91bRuyYQGkku3+MLXGd4Oo7yXDSILpSJHezdYpE64dmzbVYdkFih3FUtize4DgcL3WCnq9F/hzZue9AGScFV4K8BFVKCQiCyXjuKWB355Cg/2G7FHtwBf2oXYkWawtoJhlPcb2f9uzVORsIYDDntAUPZJqY8d9dsIYF+KPNj1N1nGSnPTB7WVGRX6b0yZhpgF7/5g5vpz1dXz7hMDgz20V3wKTof3wN3pV0aPKx6GZfu8aO0rcpdvoKL8uaKW4q+eH9/8dOazpBFKc2OL8oS0Y9OkFS9Y1n3m+EHTzk1APr3+peC2yZQmoR3mI8gkZneGiE3NkEHi5SY7Ubmb8Aw+9j39vlOvyQ+aEA9jn/Dy7UyPvvJdL6uVw5qahiWqBqnWYWORFo/ahThbAlW5Wr0D0L2rw8B6zNXk2Hu4yfnTZn2zxVevivejk5d0XqZY+rG7+DgAA///v/CY+" + return "eJzUm0Fv47oRx+/5FIN32gdk8wFyKNC+Ar30PWyb7VmgqLFMhOLokZQT76cvhqRkxbFsySvTrU8LSeH8ZvgfzlDifoVX3D9DW5fUGYn2AcArr/EZfvn2j7/Fa788AFTopFWtV2Se4S8PAAC/o7dKOpCkNUqPFWwsNYehnh4A3JasLySZjaqfYSO0wwcAixqFw2eoxQPARqGu3HMY8ysY0eBHHv75fcuPW+radOUEEP8GaGgi3lO6NbYytqSV8264esrSGWv8e9nSm4OdsIo6B8p4tEZoUGZDthH8F9HG0+ivPgag/x0jjjEr4UUpHLoPd3tcTaY+unGGmH+/UWc80OYw8NNJu51Du7rNMOhpey2RXt1eGPS0vY1FLKRWaPzqZnlsSGM/wfctOgRhh0vgt8KHC5VykowJSfQIZefhW93L+BWx5UcRGmzI7kFY6kwV//hNOBBakxQh/cjygw04AuVBCgMlgsXOpZubzndj+wRiR6rqh1BkplVQ3SpKga6P0knrmmplbmW+D4Yy0Q44LzyeEYtDu7tBTgSxpLHHYkmX7iaWwf4ysdwoSoGuj9LphdK4gv+1uum///ESTASpcIilkNsJoTDEDzK3gQgDz4T4s0Or1sdQ5utGq3rrA1Ay8vSptjbY/HxlDR72eh5XVRZpX3YPKRAe/8liy/WpiJF16geuFL3vWwQejgMowClTawSnyfdzyWbPTegIi/NgJaw/uqZEe0guTd5dQcTr1+pEYieUFmWK0zVYDTaevNBrzSGPBeXeo4vhKveziKTFCo1XQrs7CGtkfRlkJpldzZdTdFdDZpLgTL6qvIMAq3IWUia5zaTJKa6ZSJmkdJ6mxbtURzY7EyuTjhYQ5dTSAqxMeppFxJv0e8mKbS8BzCmwpWzZpbYUMKfoLrLdSXTzsXJJbT5RVoHNx8olqwtE1HnnhamUqQuLf3bo/B0ENqKARHEddCb5rcabU5yrQWeS7kLe+JrtDuKNhmejZZLoQqqcQlyIlklul6kUld3mDgILdueCZZLXIqac4loElklaF5l2whZa3aX07oQNH3wX4GWS2BVkOYV2BV4muc0hSytsa7EVFqsifEZs0NxDgmnl7VlgYPkZB/KW2XXZ71CM13Ugb8m+zD51XOPKb3zh/XVnLVsKNjm+4WL80q3IfD7Ecf0hmpNRfMX9G9ljiV9Sg2gG1n706W9St7HLI098i9CFkF7t1tL+b+GsxGhSRscSUPkt8jJpXrECT0P7N3qYbHhUVRrhTfktGOo/HsObUJ43Jp6gRGgtSXQOq0nH0vM39mwrdpwVxg+cZZeuGvKwRw81eV6Fj729MCOFFEai5o1gDhc2ZN+ErbAKfuwhWtciPUfj7BemCtPUz0g6KdLftuhaMlMyP8xMZg95OpZ4uUc/sRTv1k2al895MCTNIVtEOoh0gSkFNQNaWo71vo8qS0GkkPavDyZpSzwoYLW24WV8FMqQbYTWe5DU6QpKlNSkpYVTdCxgT1AROILOeKVBaD06wPLRIRfFJKlpNXrsT06hTauAp/4PRNIYcdVXLslqMiAMlmHWOhPKOaewahqslPCo99C50KdwJkeZfT6zc8S7Yrc3xRsiXSKaOGsM15BFvmmGFxZblK9FhVrsH3kG/Rb3YHA4BJYeiFPhCWxn4oxgAyVu4nC474+3xeDUQk0sz25XeHT+5q5/TDDbGcM6TSU0eWbRoU+ekT3h76QL4Rzj7Tw4gPcv6WOx5n5EU12zK1MBbsQ7Z2XRrXUk7XclLTmUZCpohfV9U9SId9V0zWENUA0+wb+x5bDGw7cIoaUOj4T77E9zGO/MCeWioWrFLvJ7+q7AoDwyg3TuRLvNvfEK7TYPo5xX0vHuUeKHWXReWP//0GfHdbcU8hV5uXMwHIsFT6fnLkx4zJ5CUmf8qvspM2wNX/71T64hjTCVCzMbt1gf/mvEFF1KdOGcqk3cAq5Oymp3h641nAwO9uY0IxHTokS1W22ljFw70l3Mw7g3pQ0Y9G9kX8FbsdkoCb3dBfFcOXgzIEOvMBfwXUhf8IzcSIzjJQ1ce4wGb1s0H3IHvpHztUVWsTIg2C3jRFj+H/sqFWo2F4DDPS5T+I6yCyvu2Q5jnIj3dj121qGTPBmDx2VOhfq2pk9cmQb2/r8rjDdnfR4/fipg8J+2Cqf9o6N98zd6q6JGmS+GxfTSNnYk3FsvoyN9LVhKxa5eH++vO7SixiOk0LGF/kMZ0MLFN1h8XdHE6wumW78I9Xj9m4pryW5bgHrKwywfFaOJWnTkDjcC812yKHcr8w916Ev4X48pL39NNeEAey7Oqwc11J0vvEldjrN2GTqRLVB1VsQqMqvVPuTJDbjizuoKov+hBR6+iORN6o2HvYwbbfZPLvFVZ8O+6OjNuytiLv369PDfAAAA//+cbsQq" } From 44ea4d47b73c3cae86b0b2ce64d87e7debedfbf8 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:24:51 +0200 Subject: [PATCH 22/53] Add assertion --- metricbeat/module/pgbouncer/pgbouncer_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go index 4184c51191da..8951348720e9 100644 --- a/metricbeat/module/pgbouncer/pgbouncer_test.go +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -175,14 +175,18 @@ func (m *MockModule) UnpackConfig(to interface{}) error { if m.Username == "" { return fmt.Errorf("simulated config error") } - c := to.(*struct { + c, ok := to.(*struct { Username string `config:"username"` Password string `config:"password"` }) + if !ok { + return fmt.Errorf("type assertion failed") + } c.Username = m.Username c.Password = m.Password return nil } + func TestParseURL_UnpackConfigError(t *testing.T) { mod := &MockModule{ Username: "", From c3f2a158d39d628a8230ab5f51ba620801531e0b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:26:04 +0200 Subject: [PATCH 23/53] Change formatter --- metricbeat/module/pgbouncer/pgbouncer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/pgbouncer.go b/metricbeat/module/pgbouncer/pgbouncer.go index 10fa89993d9c..f1e907bdfd1e 100644 --- a/metricbeat/module/pgbouncer/pgbouncer.go +++ b/metricbeat/module/pgbouncer/pgbouncer.go @@ -69,7 +69,7 @@ func ParseURL(mod mb.Module, rawURL string) (mb.HostData, error) { u, err := url.Parse(rawURL) if err != nil { - return mb.HostData{}, fmt.Errorf("error parsing URL: %v", err) + return mb.HostData{}, fmt.Errorf("error parsing URL: %w", err) } parse.SetURLUser(u, c.Username, c.Password) From be37fa1f4b80554fa0a3de8153c9f77ef336a444 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:26:39 +0200 Subject: [PATCH 24/53] Remove unnecessary sprintf --- metricbeat/module/pgbouncer/metricset_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/metricset_test.go b/metricbeat/module/pgbouncer/metricset_test.go index 55047d162096..d981c6c2917f 100644 --- a/metricbeat/module/pgbouncer/metricset_test.go +++ b/metricbeat/module/pgbouncer/metricset_test.go @@ -107,7 +107,7 @@ func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "pgbouncer", "metricsets": []string{"stats"}, - "hosts": []string{fmt.Sprintf("localhost:6432/pgbouncer?sslmode=disable")}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, "username": "test", "password": postgresql.GetEnvPassword(), } From 70486b6b776343b421b19f148174039fabc3e782 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:29:01 +0200 Subject: [PATCH 25/53] Add pgbouncer --- metricbeat/metricbeat.reference.yml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index cf26e8aed2e9..2eca2816b36b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -852,6 +852,38 @@ metricbeat.modules: exclude: [] #------------------------------ PGBouncer Module ------------------------------ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm From 729c2228ad52e60c45618a968ae612cfdb5abe08 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:29:27 +0200 Subject: [PATCH 26/53] Update pgbouncer --- metricbeat/modules.d/pgbouncer.yml.disabled | 22 ++++++++------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/metricbeat/modules.d/pgbouncer.yml.disabled b/metricbeat/modules.d/pgbouncer.yml.disabled index 373d7902113a..f96114664cfd 100644 --- a/metricbeat/modules.d/pgbouncer.yml.disabled +++ b/metricbeat/modules.d/pgbouncer.yml.disabled @@ -1,20 +1,14 @@ # Module: pgbouncer # Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html -# Module: pgbouncer -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html - metricbeat.modules: - module: pgbouncer - metricsets: - - stats + # metricsets: + # - stats + # - lists + # - pools + # - mem period: 10s - hosts: ["postgresql://localhost:5432"] - username: test - password: password -output.elasticsearch: - hosts: ["http://172.21.186.195:9200"] - username: elastic - password: xxxx - indices: - - index: "test" \ No newline at end of file + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + # username: test + # password: password \ No newline at end of file From c8cb4d8b6d9f351f9d6ff276f4b14e1e7f6b9e47 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 18:52:14 +0200 Subject: [PATCH 27/53] Fix import order with goimports --- metricbeat/module/pgbouncer/mem/mem_integration_test.go | 3 ++- metricbeat/module/pgbouncer/metricset_test.go | 3 ++- metricbeat/module/pgbouncer/pools/pools_integration_test.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go index 2779eb4da95c..0a8e4ba97093 100644 --- a/metricbeat/module/pgbouncer/mem/mem_integration_test.go +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -20,9 +20,10 @@ package mem import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/tests/compose" ) diff --git a/metricbeat/module/pgbouncer/metricset_test.go b/metricbeat/module/pgbouncer/metricset_test.go index d981c6c2917f..d8ffec5420f8 100644 --- a/metricbeat/module/pgbouncer/metricset_test.go +++ b/metricbeat/module/pgbouncer/metricset_test.go @@ -23,10 +23,11 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/tests/compose" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - "github.com/stretchr/testify/assert" ) func TestNewMetricSet(t *testing.T) { diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index 8a454090c301..5d8ac351859a 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -20,9 +20,10 @@ package pools import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/tests/compose" ) From 6834bc70fd43a4dd9d783e7d83daa2133c6d7be1 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 19:33:37 +0200 Subject: [PATCH 28/53] Fix import order with goimports --- metricbeat/module/pgbouncer/lists/lists_integration_test.go | 3 ++- metricbeat/module/pgbouncer/stats/stats_integration_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index 2020784d812f..c8bdcaf9a726 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -20,9 +20,10 @@ package lists import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/tests/compose" ) diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index 112d9193c67a..52ed34519a33 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -20,9 +20,10 @@ package stats import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/tests/compose" ) From 0c2195d4b17e8fbe2eb7321306bb32670794449e Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 19:58:06 +0200 Subject: [PATCH 29/53] Add generated files --- metricbeat/docs/modules/pgbouncer.asciidoc | 130 ++++++++++++++++++ .../docs/modules/pgbouncer/lists.asciidoc | 27 ++++ .../docs/modules/pgbouncer/mem.asciidoc | 27 ++++ .../docs/modules/pgbouncer/pools.asciidoc | 27 ++++ .../docs/modules/pgbouncer/stats.asciidoc | 27 ++++ metricbeat/docs/modules_list.asciidoc | 6 + 6 files changed, 244 insertions(+) create mode 100644 metricbeat/docs/modules/pgbouncer.asciidoc create mode 100644 metricbeat/docs/modules/pgbouncer/lists.asciidoc create mode 100644 metricbeat/docs/modules/pgbouncer/mem.asciidoc create mode 100644 metricbeat/docs/modules/pgbouncer/pools.asciidoc create mode 100644 metricbeat/docs/modules/pgbouncer/stats.asciidoc diff --git a/metricbeat/docs/modules/pgbouncer.asciidoc b/metricbeat/docs/modules/pgbouncer.asciidoc new file mode 100644 index 000000000000..89469e23b513 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer.asciidoc @@ -0,0 +1,130 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +:modulename: pgbouncer +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/_meta/docs.asciidoc + + +[[metricbeat-module-pgbouncer]] +== PGBouncer module + +include::{libbeat-dir}/shared/integration-link.asciidoc[] + +:modulename!: + +PgBouncer is a popular, lightweight connection pooler for PostgreSQL. This module allows you to monitor PgBouncer using Metricbeat, gathering metrics that help optimize performance and maintain stability. + +This module periodically fetches metrics from the PgBouncer poolers available at https://www.pgbouncer.org/. + +Default metricsets include: +- `lists`: Provides a list of databases and users currently connected to PgBouncer. +- `mem`: Monitors memory usage details within PgBouncer, helping detect memory leaks or spikes. +- `pools`: Tracks the status of connection pools, including active and idle connections. +- `stats`: Collects comprehensive statistics on database connections and queries processed. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use PostgreSQL URLs of the following format with PgBouncer database name included: + +[source,yaml] +----------------------------------- +[postgres://][user:pass@]host[:port]/pgbouncer[?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer:40001?sslmode=disable", "postgres://otherhost/pgbouncer:40001"] +---------------------------------------------------------------------- + +Usernames and passwords specified in the URL take precedence over those specified in the `username` and `password` configuration options. +For security best practices, consider using environment variables to pass sensitive information securely. + +[source,yaml] +---- +- module: postgresql + metricsets: ["pools"] + hosts: ["postgres://localhost/pgbouncer:6432"] + username: root + password: test +---- + +:edit_url: + +[float] +=== Example configuration + +The PGBouncer module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::pgbouncer/lists.asciidoc[] + +include::pgbouncer/mem.asciidoc[] + +include::pgbouncer/pools.asciidoc[] + +include::pgbouncer/stats.asciidoc[] + +:edit_url!: diff --git a/metricbeat/docs/modules/pgbouncer/lists.asciidoc b/metricbeat/docs/modules/pgbouncer/lists.asciidoc new file mode 100644 index 000000000000..c6fae28e91bc --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/lists.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-lists]] +=== PGBouncer lists metricset + +include::../../../module/pgbouncer/lists/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/lists/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/mem.asciidoc b/metricbeat/docs/modules/pgbouncer/mem.asciidoc new file mode 100644 index 000000000000..4369707c7fb5 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/mem.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-mem]] +=== PGBouncer mem metricset + +include::../../../module/pgbouncer/mem/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/mem/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/pools.asciidoc b/metricbeat/docs/modules/pgbouncer/pools.asciidoc new file mode 100644 index 000000000000..84032c923ddb --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/pools.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-pools]] +=== PGBouncer pools metricset + +include::../../../module/pgbouncer/pools/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/pools/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/stats.asciidoc b/metricbeat/docs/modules/pgbouncer/stats.asciidoc new file mode 100644 index 000000000000..78783d6b0ad7 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/stats.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-stats]] +=== PGBouncer stats metricset + +include::../../../module/pgbouncer/stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/stats/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index f68dc8e1e65a..c3048e4b6a51 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -251,6 +251,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] +|<> |image:./images/icon-no.png[No prebuilt dashboards] | +.4+| .4+| |<> +|<> +|<> +|<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> @@ -384,6 +389,7 @@ include::modules/nginx.asciidoc[] include::modules/openmetrics.asciidoc[] include::modules/oracle.asciidoc[] include::modules/panw.asciidoc[] +include::modules/pgbouncer.asciidoc[] include::modules/php_fpm.asciidoc[] include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] From 0518669e6231ff4418746fc42f5e4ac54e73b6f2 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 20:13:38 +0200 Subject: [PATCH 30/53] Add pgbouncer --- metricbeat/docs/fields.asciidoc | 913 ++++++++++++++++++++++++++++++++ 1 file changed, 913 insertions(+) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 783e19ffe971..a6c8eb51e67d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -70,6 +70,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -57295,6 +57296,918 @@ type: keyword -- +[[exported-fields-pgbouncer]] +== PGBouncer fields + +Metrics collected from pgbouncer. + + + +[float] +=== pgbouncer + +PGBouncer metrics. + + + +[float] +=== lists + +Shows various internal information lists. + + + +*`pgbouncer.lists.databases`*:: ++ +-- +Count of databases. + + +type: long + +-- + +*`pgbouncer.lists.users`*:: ++ +-- +Count of users. + + +type: long + +-- + +*`pgbouncer.lists.pools`*:: ++ +-- +Count of pools. + + +type: long + +-- + +*`pgbouncer.lists.free_clients`*:: ++ +-- +Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. + + +type: long + +-- + +*`pgbouncer.lists.used_clients`*:: ++ +-- +Count of used clients. + + +type: long + +-- + +*`pgbouncer.lists.login_clients`*:: ++ +-- +Count of clients in login state. + + +type: long + +-- + +*`pgbouncer.lists.free_servers`*:: ++ +-- +Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. + + +type: long + +-- + +*`pgbouncer.lists.used_servers`*:: ++ +-- +Count of used servers. + + +type: long + +-- + +*`pgbouncer.lists.dns_names`*:: ++ +-- +Count of DNS names in the cache. + + +type: long + +-- + +*`pgbouncer.lists.dns_zones`*:: ++ +-- +Count of DNS zones in the cache. + + +type: long + +-- + +*`pgbouncer.lists.dns_queries`*:: ++ +-- +Count of in-flight DNS queries. + + +type: long + +-- + +[float] +=== mem + +Shows cache memory information for various PgBouncer caches. + + + +*`pgbouncer.mem.user_cache.size`*:: ++ +-- +The size of a single slot in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.used`*:: ++ +-- +Number of used slots in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.free`*:: ++ +-- +Number of available slots in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.memtotal`*:: ++ +-- +Total bytes used by the user cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.size`*:: ++ +-- +The size of a single slot in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.used`*:: ++ +-- +Number of used slots in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.free`*:: ++ +-- +Number of available slots in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.memtotal`*:: ++ +-- +Total bytes used by the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.size`*:: ++ +-- +The size of a single slot in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.used`*:: ++ +-- +Number of used slots in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.free`*:: ++ +-- +Number of available slots in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.memtotal`*:: ++ +-- +Total bytes used by the db cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.size`*:: ++ +-- +The size of a single slot in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.used`*:: ++ +-- +Number of used slots in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.free`*:: ++ +-- +Number of available slots in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.memtotal`*:: ++ +-- +Total bytes used by the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.size`*:: ++ +-- +The size of a single slot in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.used`*:: ++ +-- +Number of used slots in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.free`*:: ++ +-- +Number of available slots in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.memtotal`*:: ++ +-- +Total bytes used by the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.size`*:: ++ +-- +The size of a single slot in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.used`*:: ++ +-- +Number of used slots in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.free`*:: ++ +-- +Number of available slots in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.memtotal`*:: ++ +-- +Total bytes used by the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.size`*:: ++ +-- +The size of a single slot in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.used`*:: ++ +-- +Number of used slots in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.free`*:: ++ +-- +Number of available slots in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.memtotal`*:: ++ +-- +Total bytes used by the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.size`*:: ++ +-- +The size of a single slot in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.used`*:: ++ +-- +Number of used slots in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.free`*:: ++ +-- +Number of available slots in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.memtotal`*:: ++ +-- +Total bytes used by the server cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.size`*:: ++ +-- +The size of a single slot in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.used`*:: ++ +-- +Number of used slots in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.free`*:: ++ +-- +Number of available slots in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.memtotal`*:: ++ +-- +Total bytes used by the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.size`*:: ++ +-- +The size of a single slot in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.used`*:: ++ +-- +Number of used slots in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.free`*:: ++ +-- +Number of available slots in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.memtotal`*:: ++ +-- +Total bytes used by the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.size`*:: ++ +-- +The size of a single slot in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.used`*:: ++ +-- +Number of used slots in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.free`*:: ++ +-- +Number of available slots in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.memtotal`*:: ++ +-- +Total bytes used by the server prepared statement cache. + + +type: long + +-- + +[float] +=== pools + +Shows the current state of the connection pools. + + + +*`pgbouncer.pools.database`*:: ++ +-- +Name of the database. + + +type: keyword + +-- + +*`pgbouncer.pools.user`*:: ++ +-- +Name of the user. + + +type: keyword + +-- + +*`pgbouncer.pools.cl_active`*:: ++ +-- +Client connections that are either linked to server connections or are idle with no queries waiting to be processed. + + +type: long + +-- + +*`pgbouncer.pools.cl_waiting`*:: ++ +-- +Client connections that have sent queries but have not yet got a server connection. + + +type: long + +-- + +*`pgbouncer.pools.cl_active_cancel_req`*:: ++ +-- +Client connections that have forwarded query cancellations to the server and are waiting for the server response. + + +type: long + +-- + +*`pgbouncer.pools.cl_waiting_cancel_req`*:: ++ +-- +Client connections that have not forwarded query cancellations to the server yet. + + +type: long + +-- + +*`pgbouncer.pools.sv_active`*:: ++ +-- +Server connections that are linked to a client. + + +type: long + +-- + +*`pgbouncer.pools.sv_active_cancel`*:: ++ +-- +Server connections that are currently forwarding a cancel request. + + +type: long + +-- + +*`pgbouncer.pools.sv_being_canceled`*:: ++ +-- +Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. + + +type: long + +-- + +*`pgbouncer.pools.sv_idle`*:: ++ +-- +Server connections that are unused and immediately usable for client queries. + + +type: long + +-- + +*`pgbouncer.pools.sv_used`*:: ++ +-- +Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. + + +type: long + +-- + +*`pgbouncer.pools.sv_tested`*:: ++ +-- +Server connections that are currently running either server_reset_query or server_check_query. + + +type: long + +-- + +*`pgbouncer.pools.sv_login`*:: ++ +-- +Server connections currently in the process of logging in. + + +type: long + +-- + +*`pgbouncer.pools.maxwait_us`*:: ++ +-- +Microsecond part of the maximum waiting time. Represents the total wait time in microseconds. + + +type: long + +-- + +*`pgbouncer.pools.pool_mode`*:: ++ +-- +The pooling mode in use. + + +type: keyword + +-- + +[float] +=== stats + +Shows statistics since the process start. + + + +*`pgbouncer.stats.database`*:: ++ +-- +Name of the database this backend is connected to. + + +type: keyword + +-- + +*`pgbouncer.stats.total_query_count`*:: ++ +-- +Total number of SQL commands pooled by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.total_server_assignment_count`*:: ++ +-- +Total times a server was assigned to a client. + + +type: long + +-- + +*`pgbouncer.stats.total_received`*:: ++ +-- +Total volume in bytes of network traffic received by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.total_sent`*:: ++ +-- +Total volume in bytes of network traffic sent by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.total_xact_time`*:: ++ +-- +Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. + + +type: long + +-- + +*`pgbouncer.stats.total_query_time`*:: ++ +-- +Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. + + +type: long + +-- + +*`pgbouncer.stats.total_wait_time`*:: ++ +-- +Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. + + +type: long + +-- + +*`pgbouncer.stats.total_xact_count`*:: ++ +-- +Total number of SQL transactions pooled by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.avg_xact_count`*:: ++ +-- +Average transactions per second in last stat period. + + +type: long + +-- + +*`pgbouncer.stats.avg_query_count`*:: ++ +-- +Average queries per second in last stat period. + + +type: long + +-- + +*`pgbouncer.stats.avg_server_assignment_count`*:: ++ +-- +Average number of times a server as assigned to a client per second in the last stat period. + + +type: long + +-- + +*`pgbouncer.stats.avg_recv`*:: ++ +-- +Average received (from clients) bytes per second. + + +type: long + +-- + +*`pgbouncer.stats.avg_sent`*:: ++ +-- +Average sent (to clients) bytes per second. + + +type: long + +-- + +*`pgbouncer.stats.avg_xact_time`*:: ++ +-- +Average transaction duration, in microseconds. + + +type: long + +-- + +*`pgbouncer.stats.avg_query_time`*:: ++ +-- +Average query duration, in microseconds. + + +type: long + +-- + +*`pgbouncer.stats.avg_wait_time`*:: ++ +-- +Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). + + +type: long + +-- + [[exported-fields-php_fpm]] == PHP_FPM fields From ee77e6689420e4164d61e3799cc25b637b53c3b8 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 20:31:13 +0200 Subject: [PATCH 31/53] Add pgbouncer --- x-pack/metricbeat/metricbeat.reference.yml | 102 ++++++++++++++------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 240acb2cfd6a..65a29d2e77fd 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1304,6 +1304,40 @@ metricbeat.modules: hosts: ["localhost"] +#------------------------------ PGBouncer Module ------------------------------ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass + #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm metricsets: @@ -1345,9 +1379,11 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass -#----------------------- Prometheus Typed Metrics Module ----------------------- +#------------------------------ Prometheus Module ------------------------------ +# Metrics collected from a Prometheus endpoint - module: prometheus period: 10s + metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1356,20 +1392,14 @@ metricbeat.modules: #username: "user" #password: "secret" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1377,24 +1407,12 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Secure settings for the server using TLS/SSL: - #ssl.certificate: "/etc/pki/server/cert.pem" - #ssl.key: "/etc/pki/server/cert.key" - # Count number of metrics present in Elasticsearch document (default: false) #metrics_count: false - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true - - # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns - #types_patterns: - # counter_patterns: [] - # histogram_patterns: [] + # Secure settings for the server using TLS/SSL: + #ssl.certificate: "/etc/pki/server/cert.pem" + #ssl.key: "/etc/pki/server/cert.key" # Metrics that will be collected using a PromQL #- module: prometheus @@ -1422,11 +1440,9 @@ metricbeat.modules: # params: # query: "some_value" -#------------------------------ Prometheus Module ------------------------------ -# Metrics collected from a Prometheus endpoint +#----------------------- Prometheus Typed Metrics Module ----------------------- - module: prometheus period: 10s - metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1435,14 +1451,20 @@ metricbeat.modules: #username: "user" #password: "secret" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1450,13 +1472,25 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + # Metrics that will be collected using a PromQL #- module: prometheus # metricsets: ["query"] From c72880a60f76a0c27c8bb547b2c83e30f5b12217 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 22:28:56 +0200 Subject: [PATCH 32/53] Use testing version --- .../module/pgbouncer/docker-compose.yml | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml index 76ce86c9d90e..4fbd02cba837 100644 --- a/metricbeat/module/pgbouncer/docker-compose.yml +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -27,20 +27,23 @@ services: start_period: 15s elasticsearch: - image: elasticsearch:8.11.0 hostname: elasticsearch - environment: - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "http.host=0.0.0.0" - - "xpack.security.enabled=false" - - "discovery.type=single-node" - # - "transport.host=127.0.0.1" + extends: + file: ../../../testing/environments/latest.yml + service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s ports: - 9200:9200 kibana: - image: kibana:8.11.0 - environment: - - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + extends: + file: ../../../testing/environments/latest.yml + service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 ports: - 5601:5601 \ No newline at end of file From 213faf018bace6cbee1c276a87fc4d27a5132ff8 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 8 Oct 2024 22:29:56 +0200 Subject: [PATCH 33/53] Refactor fetch method --- metricbeat/module/pgbouncer/mem/mem.go | 27 ++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index a6c11748bed4..6ca665c34d7f 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -55,20 +55,43 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format // It publishes the event which is then forwarded to the output. In case of an error, an error is reported. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + // Create a new context for this operation. ctx := context.Background() + + // Execute the "SHOW MEM;" query against the database. results, err := m.QueryStats(ctx, "SHOW MEM;") if err != nil { + // Return the error if the query fails. return fmt.Errorf("error in QueryStats: %w", err) } + // Initialize an empty map to store aggregated results. + data := mapstr.M{} + + // Iterate over each result from the query. for _, result := range results { - var data mapstr.M + // Apply the predefined schema to the result to format it properly. + tmpData, err := schema.Apply(result) + if err != nil { + // Log the error and skip this iteration if schema application fails. + log.Printf("Error applying schema: %v", err) + continue + } - data, _ = schema.Apply(result) + // Aggregate the formatted data into the data map. + for k, v := range tmpData { + data[k] = v + } + } + // Check if there is any data collected. + if len(data) > 0 { + // Create and report an event with the collected data. reporter.Event(mb.Event{ MetricSetFields: data, }) } + + // Return nil to indicate successful completion. return nil } From 9e63bbf433540c612cc371a9b1d78bbfb920f32f Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Thu, 10 Oct 2024 16:24:15 +0200 Subject: [PATCH 34/53] Refactor error logging --- metricbeat/module/pgbouncer/mem/mem.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index 6ca665c34d7f..cec5a17c388c 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -74,7 +74,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { tmpData, err := schema.Apply(result) if err != nil { // Log the error and skip this iteration if schema application fails. - log.Printf("Error applying schema: %v", err) + fmt.Errorf("Error applying schema: %w", err) continue } From 5e8158dae00b5aa5942c652b3c1331dc55124b55 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Thu, 10 Oct 2024 16:25:57 +0200 Subject: [PATCH 35/53] Copy config files in build instead of mounting --- metricbeat/module/pgbouncer/_meta/Dockerfile | 5 +++++ metricbeat/module/pgbouncer/docker-compose.yml | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 metricbeat/module/pgbouncer/_meta/Dockerfile diff --git a/metricbeat/module/pgbouncer/_meta/Dockerfile b/metricbeat/module/pgbouncer/_meta/Dockerfile new file mode 100644 index 000000000000..e729feb0530c --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/Dockerfile @@ -0,0 +1,5 @@ +FROM edoburu/pgbouncer +COPY ./pgbouncer.ini /etc/pgbouncer/pgbouncer.ini +COPY ./userlist.txt /etc/pgbouncer/userlist.txt +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/pgbouncer", "/etc/pgbouncer/pgbouncer.ini"] \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml index 4fbd02cba837..2d31c13945fa 100644 --- a/metricbeat/module/pgbouncer/docker-compose.yml +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -12,13 +12,12 @@ services: - POSTGRES_INITDB_ARGS=--auth=md5 pgbouncer: - image: edoburu/pgbouncer + build: + context: ./_meta + dockerfile: Dockerfile hostname: pgbouncer ports: - 6432:6432 - volumes: - - ./_meta/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini - - ./_meta/userlist.txt:/etc/pgbouncer/userlist.txt healthcheck: test: ['CMD-SHELL', 'psql -h localhost -p 6432 -U test -d pgbouncer -c "SHOW STATS"'] interval: 15s From 663b500b87ba981d59d9eb44ee5adb30928ba320 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Thu, 10 Oct 2024 16:37:14 +0200 Subject: [PATCH 36/53] Revert "Refactor error logging" This reverts commit 9e63bbf433540c612cc371a9b1d78bbfb920f32f. --- metricbeat/module/pgbouncer/mem/mem.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index cec5a17c388c..6ca665c34d7f 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -74,7 +74,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { tmpData, err := schema.Apply(result) if err != nil { // Log the error and skip this iteration if schema application fails. - fmt.Errorf("Error applying schema: %w", err) + log.Printf("Error applying schema: %v", err) continue } From fdd7d0f9b03b9c16a71c43d1934e4abe0b05834c Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Fri, 11 Oct 2024 13:28:18 +0200 Subject: [PATCH 37/53] Add build tags and fix convention name --- metricbeat/module/pgbouncer/lists/lists_integration_test.go | 2 ++ metricbeat/module/pgbouncer/mem/mem_integration_test.go | 2 ++ .../{metricset_test.go => metricset_integration_test.go} | 2 ++ metricbeat/module/pgbouncer/pools/pools_integration_test.go | 2 ++ metricbeat/module/pgbouncer/stats/stats_integration_test.go | 2 ++ 5 files changed, 10 insertions(+) rename metricbeat/module/pgbouncer/{metricset_test.go => metricset_integration_test.go} (99%) diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index c8bdcaf9a726..0076a646e496 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build integration + package lists import ( diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go index 0a8e4ba97093..46e81dba823b 100644 --- a/metricbeat/module/pgbouncer/mem/mem_integration_test.go +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build integration + package mem import ( diff --git a/metricbeat/module/pgbouncer/metricset_test.go b/metricbeat/module/pgbouncer/metricset_integration_test.go similarity index 99% rename from metricbeat/module/pgbouncer/metricset_test.go rename to metricbeat/module/pgbouncer/metricset_integration_test.go index d8ffec5420f8..7a80ac2b93d5 100644 --- a/metricbeat/module/pgbouncer/metricset_test.go +++ b/metricbeat/module/pgbouncer/metricset_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build integration + package pgbouncer import ( diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index 5d8ac351859a..f1363c50a252 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build integration + package pools import ( diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index 52ed34519a33..0ec224d77229 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build integration + package stats import ( From 2222740594bb131df1fd1ecdc9088f147573a28d Mon Sep 17 00:00:00 2001 From: Manuel Saks <37194183+manuelsaks@users.noreply.github.com> Date: Tue, 10 Dec 2024 20:03:05 +0100 Subject: [PATCH 38/53] Refactor: Inline resultMap assignment Co-authored-by: Kush Rana <89848966+kush-elastic@users.noreply.github.com> --- metricbeat/module/pgbouncer/lists/lists.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 71559303c196..91b24ff8cdd9 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -65,9 +65,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if !ok { return fmt.Errorf("expected string type for 'list' but got something else") } - key := listValue - value := s["items"] - resultMap[key] = value + resultMap[listValue] = s["items"] } data, err := schema.Apply(resultMap) if err != nil { From 79260f5b08c5b61e4cefed6d7be670d136a577b2 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 10 Dec 2024 20:05:17 +0100 Subject: [PATCH 39/53] Normalize fields to json --- .../module/pgbouncer/lists/_meta/fields.yml | 10 +++---- metricbeat/module/pgbouncer/lists/data.go | 29 +++++++++++-------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/_meta/fields.yml b/metricbeat/module/pgbouncer/lists/_meta/fields.yml index 1f383003bed2..ada09b863265 100644 --- a/metricbeat/module/pgbouncer/lists/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/lists/_meta/fields.yml @@ -16,23 +16,23 @@ type: long description: > Count of pools. - - name: free_clients + - name: clients.free type: long description: > Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. - - name: used_clients + - name: clients.used type: long description: > Count of used clients. - - name: login_clients + - name: clients.login type: long description: > Count of clients in login state. - - name: free_servers + - name: servers.free type: long description: > Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. - - name: used_servers + - name: servers.used type: long description: > Count of used servers. diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 111871cc6174..6291f882606b 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -23,16 +23,21 @@ import ( ) var schema = s.Schema{ - "databases": c.Int("databases"), - "users": c.Int("users"), - "peers": c.Int("peers"), - "pools": c.Int("pools"), - "peer_pools": c.Int("peer_pools"), - "free_clients": c.Int("free_clients"), - "used_clients": c.Int("used_clients"), - "login_clients": c.Int("login_clients"), - "free_servers": c.Int("free_servers"), - "used_servers": c.Int("used_servers"), - "dns_names": c.Int("dns_names"), - "dns_zones": c.Int("dns_zones"), + "databases": c.Int("databases"), + "users": c.Int("users"), + "peers": c.Int("peers"), + "pools": c.Int("pools"), + "peer_pools": c.Int("peer_pools"), + "clients": s.Object{ + "free": c.Int("free_clients"), + "used": c.Int("used_clients"), + "login": c.Int("login_clients"), + }, + "servers": s.Object{ + "free": c.Int("free_servers"), + "used": c.Int("used_servers"), + }, + "dns_names": c.Int("dns_names"), + "dns_zones": c.Int("dns_zones"), + "dns_queries": c.Int("dns_queries"), } From aba15595f4906db68b52315e2ea1e31ac551c53f Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 10 Dec 2024 20:26:25 +0100 Subject: [PATCH 40/53] Refactor: Use context in Fetch method --- metricbeat/module/pgbouncer/lists/lists.go | 3 +-- metricbeat/module/pgbouncer/pools/pools.go | 3 +-- metricbeat/module/pgbouncer/stats/stats.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index 91b24ff8cdd9..bd2f29ed9335 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -53,8 +53,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format // It publishes the event which is then forwarded to the output. In case of an error, an error is reported. -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - ctx := context.Background() +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { results, err := m.QueryStats(ctx, "SHOW LISTS;") if err != nil { return fmt.Errorf("error in QueryStats: %w", err) diff --git a/metricbeat/module/pgbouncer/pools/pools.go b/metricbeat/module/pgbouncer/pools/pools.go index 3f9d599641a1..be489de2b342 100644 --- a/metricbeat/module/pgbouncer/pools/pools.go +++ b/metricbeat/module/pgbouncer/pools/pools.go @@ -50,8 +50,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format // It publishes the event which is then forwarded to the output. In case of an error, an error is reported. -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - ctx := context.Background() +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { results, err := m.QueryStats(ctx, "SHOW POOLS;") if err != nil { return fmt.Errorf("error in QueryStats: %w", err) diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go index 05594abcc8c5..bad9514622d6 100644 --- a/metricbeat/module/pgbouncer/stats/stats.go +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -50,8 +50,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format // It publishes the event which is then forwarded to the output. In case of an error, an error is reported. -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - ctx := context.Background() +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { results, err := m.QueryStats(ctx, "SHOW STATS;") if err != nil { return fmt.Errorf("error in QueryStats: %w", err) From 7973937dbd0543266fb944629a52fe4419e1b22c Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 10 Dec 2024 21:49:07 +0100 Subject: [PATCH 41/53] Refactor: Use context in Fetch method --- .../pgbouncer/lists/lists_integration_test.go | 13 ++++++------- .../module/pgbouncer/mem/mem_integration_test.go | 7 +++---- .../pgbouncer/pools/pools_integration_test.go | 7 +++---- .../pgbouncer/stats/stats_integration_test.go | 7 +++---- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index 0076a646e496..20681a1e8f18 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" @@ -35,19 +36,17 @@ func TestMetricSet_Fetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields assert.Contains(t, event, "databases") assert.Contains(t, event, "users") assert.Contains(t, event, "peers") assert.Contains(t, event, "pools") assert.Contains(t, event, "peer_pools") - assert.Contains(t, event, "used_clients") - assert.Contains(t, event, "free_servers") - assert.Contains(t, event, "used_servers") + assert.Contains(t, event, "clients.used") + assert.Contains(t, event, "servers.free") + assert.Contains(t, event, "servers.used") } func getConfig(host string) map[string]interface{} { return map[string]interface{}{ diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go index 46e81dba823b..9dabbc765cd8 100644 --- a/metricbeat/module/pgbouncer/mem/mem_integration_test.go +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" @@ -35,10 +36,8 @@ func TestMetricSet_Fetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields assert.Contains(t, event["user_cache"], "size") assert.Contains(t, event["user_cache"], "used") diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index f1363c50a252..255b79f0c21c 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" @@ -35,10 +36,8 @@ func TestMetricSet_Fetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields assert.Contains(t, event, "user") assert.Contains(t, event, "cl_active") diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index 0ec224d77229..cd03ff2a347a 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" @@ -35,10 +36,8 @@ func TestMetricSet_Fetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields assert.Contains(t, event, "total_xact_count") assert.Contains(t, event, "total_server_assignment_count") From 0c5ed2da68f27774e09445c77fd74bcd014f4712 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 10 Dec 2024 21:55:20 +0100 Subject: [PATCH 42/53] update normalized data --- .../module/pgbouncer/lists/_meta/data.json | 25 +++++++++++-------- .../pgbouncer/lists/lists_integration_test.go | 6 ++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/_meta/data.json b/metricbeat/module/pgbouncer/lists/_meta/data.json index 49bf14137359..0a5a1404b8f8 100644 --- a/metricbeat/module/pgbouncer/lists/_meta/data.json +++ b/metricbeat/module/pgbouncer/lists/_meta/data.json @@ -11,18 +11,23 @@ }, "pgbouncer": { "lists": { - "databases": 1, - "free_clients": 48, - "free_servers": 0, - "users": 4, - "dns_zones": 0, + "users": 2, + "clients": { + "free": 47, + "used": 3, + "login": 0 + }, "peers": 0, "peer_pools": 0, - "login_clients": 0, - "pools": 1, - "used_clients": 2, - "used_servers": 0, - "dns_names": 0 + "dns_queries": 0, + "databases": 1, + "dns_names": 0, + "servers": { + "free": 0, + "used": 0 + }, + "dns_zones": 0, + "pools": 1 } }, "service": { diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index 20681a1e8f18..c2fc18093259 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -44,9 +44,9 @@ func TestMetricSet_Fetch(t *testing.T) { assert.Contains(t, event, "peers") assert.Contains(t, event, "pools") assert.Contains(t, event, "peer_pools") - assert.Contains(t, event, "clients.used") - assert.Contains(t, event, "servers.free") - assert.Contains(t, event, "servers.used") + assert.Contains(t, event["clients"], "used") + assert.Contains(t, event["servers"], "free") + assert.Contains(t, event["servers"], "used") } func getConfig(host string) map[string]interface{} { return map[string]interface{}{ From 2046288a4af72abfa4ef1a9e624eb99592e57e6f Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Tue, 10 Dec 2024 22:16:58 +0100 Subject: [PATCH 43/53] Refactor: Use context in Fetch method --- metricbeat/module/pgbouncer/mem/mem.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index 6ca665c34d7f..f0cc64807797 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -54,10 +54,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format // It publishes the event which is then forwarded to the output. In case of an error, an error is reported. -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - // Create a new context for this operation. - ctx := context.Background() - +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Execute the "SHOW MEM;" query against the database. results, err := m.QueryStats(ctx, "SHOW MEM;") if err != nil { From 1123f1294da8fffcac170b176ef9eab10fe6f044 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 15 Dec 2024 12:31:59 +0100 Subject: [PATCH 44/53] Change to metricset logger --- metricbeat/module/pgbouncer/mem/mem.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index f0cc64807797..b950e7abbacc 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -71,7 +71,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { tmpData, err := schema.Apply(result) if err != nil { // Log the error and skip this iteration if schema application fails. - log.Printf("Error applying schema: %v", err) + m.Logger().Errorf("Error applying schema: %v", err) continue } From 5971098919abdcc02ad40553775277218294109b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 15 Dec 2024 12:37:41 +0100 Subject: [PATCH 45/53] Update fields file --- metricbeat/module/pgbouncer/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go index ebc6b711fce0..e693182bb5d0 100644 --- a/metricbeat/module/pgbouncer/fields.go +++ b/metricbeat/module/pgbouncer/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPgbouncer returns asset data. // This is the base64 encoded zlib format compressed contents of module/pgbouncer. func AssetPgbouncer() string { - return "eJzUm0Fv47oRx+/5FIN32gdk8wFyKNC+Ar30PWyb7VmgqLFMhOLokZQT76cvhqRkxbFsySvTrU8LSeH8ZvgfzlDifoVX3D9DW5fUGYn2AcArr/EZfvn2j7/Fa788AFTopFWtV2Se4S8PAAC/o7dKOpCkNUqPFWwsNYehnh4A3JasLySZjaqfYSO0wwcAixqFw2eoxQPARqGu3HMY8ysY0eBHHv75fcuPW+radOUEEP8GaGgi3lO6NbYytqSV8264esrSGWv8e9nSm4OdsIo6B8p4tEZoUGZDthH8F9HG0+ivPgag/x0jjjEr4UUpHLoPd3tcTaY+unGGmH+/UWc80OYw8NNJu51Du7rNMOhpey2RXt1eGPS0vY1FLKRWaPzqZnlsSGM/wfctOgRhh0vgt8KHC5VykowJSfQIZefhW93L+BWx5UcRGmzI7kFY6kwV//hNOBBakxQh/cjygw04AuVBCgMlgsXOpZubzndj+wRiR6rqh1BkplVQ3SpKga6P0knrmmplbmW+D4Yy0Q44LzyeEYtDu7tBTgSxpLHHYkmX7iaWwf4ysdwoSoGuj9LphdK4gv+1uum///ESTASpcIilkNsJoTDEDzK3gQgDz4T4s0Or1sdQ5utGq3rrA1Ay8vSptjbY/HxlDR72eh5XVRZpX3YPKRAe/8liy/WpiJF16geuFL3vWwQejgMowClTawSnyfdzyWbPTegIi/NgJaw/uqZEe0guTd5dQcTr1+pEYieUFmWK0zVYDTaevNBrzSGPBeXeo4vhKveziKTFCo1XQrs7CGtkfRlkJpldzZdTdFdDZpLgTL6qvIMAq3IWUia5zaTJKa6ZSJmkdJ6mxbtURzY7EyuTjhYQ5dTSAqxMeppFxJv0e8mKbS8BzCmwpWzZpbYUMKfoLrLdSXTzsXJJbT5RVoHNx8olqwtE1HnnhamUqQuLf3bo/B0ENqKARHEddCb5rcabU5yrQWeS7kLe+JrtDuKNhmejZZLoQqqcQlyIlklul6kUld3mDgILdueCZZLXIqac4loElklaF5l2whZa3aX07oQNH3wX4GWS2BVkOYV2BV4muc0hSytsa7EVFqsifEZs0NxDgmnl7VlgYPkZB/KW2XXZ71CM13Ugb8m+zD51XOPKb3zh/XVnLVsKNjm+4WL80q3IfD7Ecf0hmpNRfMX9G9ljiV9Sg2gG1n706W9St7HLI098i9CFkF7t1tL+b+GsxGhSRscSUPkt8jJpXrECT0P7N3qYbHhUVRrhTfktGOo/HsObUJ43Jp6gRGgtSXQOq0nH0vM39mwrdpwVxg+cZZeuGvKwRw81eV6Fj729MCOFFEai5o1gDhc2ZN+ErbAKfuwhWtciPUfj7BemCtPUz0g6KdLftuhaMlMyP8xMZg95OpZ4uUc/sRTv1k2al895MCTNIVtEOoh0gSkFNQNaWo71vo8qS0GkkPavDyZpSzwoYLW24WV8FMqQbYTWe5DU6QpKlNSkpYVTdCxgT1AROILOeKVBaD06wPLRIRfFJKlpNXrsT06hTauAp/4PRNIYcdVXLslqMiAMlmHWOhPKOaewahqslPCo99C50KdwJkeZfT6zc8S7Yrc3xRsiXSKaOGsM15BFvmmGFxZblK9FhVrsH3kG/Rb3YHA4BJYeiFPhCWxn4oxgAyVu4nC474+3xeDUQk0sz25XeHT+5q5/TDDbGcM6TSU0eWbRoU+ekT3h76QL4Rzj7Tw4gPcv6WOx5n5EU12zK1MBbsQ7Z2XRrXUk7XclLTmUZCpohfV9U9SId9V0zWENUA0+wb+x5bDGw7cIoaUOj4T77E9zGO/MCeWioWrFLvJ7+q7AoDwyg3TuRLvNvfEK7TYPo5xX0vHuUeKHWXReWP//0GfHdbcU8hV5uXMwHIsFT6fnLkx4zJ5CUmf8qvspM2wNX/71T64hjTCVCzMbt1gf/mvEFF1KdOGcqk3cAq5Oymp3h641nAwO9uY0IxHTokS1W22ljFw70l3Mw7g3pQ0Y9G9kX8FbsdkoCb3dBfFcOXgzIEOvMBfwXUhf8IzcSIzjJQ1ce4wGb1s0H3IHvpHztUVWsTIg2C3jRFj+H/sqFWo2F4DDPS5T+I6yCyvu2Q5jnIj3dj121qGTPBmDx2VOhfq2pk9cmQb2/r8rjDdnfR4/fipg8J+2Cqf9o6N98zd6q6JGmS+GxfTSNnYk3FsvoyN9LVhKxa5eH++vO7SixiOk0LGF/kMZ0MLFN1h8XdHE6wumW78I9Xj9m4pryW5bgHrKwywfFaOJWnTkDjcC812yKHcr8w916Ev4X48pL39NNeEAey7Oqwc11J0vvEldjrN2GTqRLVB1VsQqMqvVPuTJDbjizuoKov+hBR6+iORN6o2HvYwbbfZPLvFVZ8O+6OjNuytiLv369PDfAAAA//+cbsQq" + return "eJzUm81u4zgSx+95isKceoB0HiCHBfYD2MvOoHfTezYoqiwToVgaknLifvpFkfqKY9mSm6Z3fAokmfUr8l8flJmv8IqHZ2iqgloj0T4AeOU1PsMv3/75t3jtlweAEp20qvGKzDP85QEA4Df0VkkHkrRG6bGEraV6HOrpAcDtyPqNJLNV1TNshXb4AGBRo3D4DJV4ANgq1KV7DmN+BSNq/MjDH39o+HFLbdNdOQHEnwEa6oj31N2aWpla0sp5N1w9ZemMNf687OjNwV5YRa0DZTxaIzQosyVbC/5GtPE0+dbHCeg/x4hTzFJ4UQiH7sPdHleTqY5unCHmz9+pNR5oOw78dNJu69AmtxkGPW2vIdLJ7YVBT9uTWqHx7mlrEVOb5TEHA/B9hw5B2OES+J3w4UKpnCRjQhA9QtF6+Fb1Mn5FbPhRhBprsgcQllpTxi+/CQdCa5IihB9ZfrAGR6A8SGGgQLDYuu7mtvXt1D6B2JMq+yEUmQuzxCPdQAzlYOCsdU2VMqnN95OhDITxwXnh8TSIQ7tn5d5MLL2BiVi6S3cTy2B/sVh6J24mlt7A6URp3Ib/Sp5E/vH7SzARpMJTLIXczQiFIX6QuQ1EGHghxB8tWpUeQ5mvW62qnQ9AnZGnT7W1xvrnK2vwsNfztKqySPuyO4ZAePwniy3Xp02cWad+pAr17zsEHo4nUIBTptIITpPv15LNnlvQCVbC2Pq9rQu0Y3Bp8u4KooQ5cSQSe6G0KLp5ugarxtqTFzrVGvJYUBw8ujhdxWERkbRYovFKaHcHYU2sr4PMJLOr+XKK7mrITBJcyFcWdxBgWSxCyiS3hTQ5xbUQKZOUztM0eJfqyGYXYmXS0QqinFpagZVJT4uIeJN+L1mx7TWAOQW2li271NYC5hTdRbY7iW45Vi6pLSfKKrDlWLlkdYGIWu+8MKUy1cbiHy06fweBTSigo7gOOpP8kvHmFGcy6EzSXckbX7PdQbzR8GK0TBJdSZVTiCvRMsntMpWiot3eQWDB7lKwTPJaxZRTXKvAMknrItNe2I1Wdym9e2HDD74r8DJJ7AqynEK7Ai+T3JaQdRm2sdgIi+Um/IxYo7mHBLvM27PAwPIzDuQts2nZ71CM0zqQt2RfZp87rnHlb3zh/XVrLVsKNnl+w8X4S7ci8/kQx/WHaE7O4ise3sgeS/ySGkQ9sPajz/8mdRu7PPLcwYmNkF7tkx1WCGclJosyOZaAyu+Q06R5xRI8De3f5GGy4VFVaoQ35XdgqP/xGN6E8rwx8QQFQmNJonNYzjrWPX9jz3Ziz1Fh/MBZtN1VQx4O6KEiz1n42NsLK7KRwkjUvBHM4cKW7JuwJZbBjwNE61p0z9E0+oUpwzL1K9KdFOlvW3QNmTmZjyuT2UNejjVeHtDPpOJ92qB5+RwHQ9CM0SK6g0gXmLpJzYDWpWN96GeVpSC6Ke1fH8zSFjgqIFnb8DI9CmXI1kLrA0hqdQkFSqq71MIhOhWwJygJHEFrvNIgtJ4cYPnokItiklQ3Gj32J6fQdlnAU/8F0WmMuOor18lqdkIYLMOqtSaUcw5hVddYKuFRH6B1oU/hSI4y+3xm54g3Ybc3xxtmukA0cdUYriaLfNMMLyx2KF83JWpxeOQV9Ds8gMHhEFj3QFwKT2BbE1cEayhwG4fDQ3+8LU5OJdRMenb7jUfnb+76xwCzrTGs066Edp5ZdOg7z8ie8HfWhZTnJE94MIL3L+ljseZ+RFNVsStzE1yLd47KTZvqSNpvSlpyKMmU0Ajr+6aoFu+qbusxB6gan+A/2PC0xsO3CKGlDo+E++xPPY535oTypqYyYRf5vftdgUF5ZAZp3Yl2m3vjBO02D6OcV9Lx7lHih1V0Xlj/Z+izY94thHxFTncOhmOx4On02oUFj9GzkdQan3Q/ZYat4cu//8U1pBamdGFl4xbrw79GzNF1gS6cU5WJW8DkpKx2N3at4WRwsLekGYmYFiWqfbJMGbn2pNsYh3FvSlsw6N/IvoK3YrtVEnq7K+Yz8eQtgAy9wlLAdyH9hlfkRmKcpjRwzTEavO3QfIgd+EbOVxZZxcqAYLeMEyH9P/ZVKtRsLgDjPS5T+I6yDRn3bIcxDcR7ux4769BJnpyDx3VOhfqW0ieuTAN7/+8K081ZH8ePnwoY/Lcpw2n/6Gjf/E3eqqhJ5IshmV7axk6Ee+s0OtHXilQq9lV6vL/u0YoKj5BCxxb6D2VACxffYPF1RTOvL5gufRHq8fo3FdeS3bYA9ZTjKh8Vo5ladOQONwLLXbIo94n5hzr0JfzXYxeXv3Y1YYQ9N8/JJzXUnS+8SV2Pk7oMnYgWKFsrYhVZ1GqPcXIDrrizuoLo/yjBwxfRedP1xsNexk02+ydTfNnasC86evPuNjGWfn16+F8AAAD//5GzwzU=" } From f470c86c37c011562914860cb01a933164fc3252 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 15 Dec 2024 13:04:10 +0100 Subject: [PATCH 46/53] Update fields --- metricbeat/docs/fields.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 53407f680208..eaef6fe9d014 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -57347,7 +57347,7 @@ type: long -- -*`pgbouncer.lists.free_clients`*:: +*`pgbouncer.lists.clients.free`*:: + -- Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. @@ -57357,7 +57357,7 @@ type: long -- -*`pgbouncer.lists.used_clients`*:: +*`pgbouncer.lists.clients.used`*:: + -- Count of used clients. @@ -57367,7 +57367,7 @@ type: long -- -*`pgbouncer.lists.login_clients`*:: +*`pgbouncer.lists.clients.login`*:: + -- Count of clients in login state. @@ -57377,7 +57377,7 @@ type: long -- -*`pgbouncer.lists.free_servers`*:: +*`pgbouncer.lists.servers.free`*:: + -- Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. @@ -57387,7 +57387,7 @@ type: long -- -*`pgbouncer.lists.used_servers`*:: +*`pgbouncer.lists.servers.used`*:: + -- Count of used servers. From de66b87a07dc684ba2e4ed085e55fc65d6db3a3e Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 15 Dec 2024 13:47:33 +0100 Subject: [PATCH 47/53] update --- x-pack/metricbeat/metricbeat.reference.yml | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4870d37c2a43..52e6799ba682 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1442,11 +1442,9 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass -#------------------------------ Prometheus Module ------------------------------ -# Metrics collected from a Prometheus endpoint +#----------------------- Prometheus Typed Metrics Module ----------------------- - module: prometheus period: 10s - metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1455,14 +1453,20 @@ metricbeat.modules: #username: "user" #password: "secret" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1470,13 +1474,25 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + # Metrics that will be collected using a PromQL #- module: prometheus # metricsets: ["query"] @@ -1503,9 +1519,11 @@ metricbeat.modules: # params: # query: "some_value" -#----------------------- Prometheus Typed Metrics Module ----------------------- +#------------------------------ Prometheus Module ------------------------------ +# Metrics collected from a Prometheus endpoint - module: prometheus period: 10s + metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1514,20 +1532,14 @@ metricbeat.modules: #username: "user" #password: "secret" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1535,24 +1547,12 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Secure settings for the server using TLS/SSL: - #ssl.certificate: "/etc/pki/server/cert.pem" - #ssl.key: "/etc/pki/server/cert.key" - # Count number of metrics present in Elasticsearch document (default: false) #metrics_count: false - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true - - # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns - #types_patterns: - # counter_patterns: [] - # histogram_patterns: [] + # Secure settings for the server using TLS/SSL: + #ssl.certificate: "/etc/pki/server/cert.pem" + #ssl.key: "/etc/pki/server/cert.key" # Metrics that will be collected using a PromQL #- module: prometheus From 5304c508089be400e5b42ffa7148a601631b7aed Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 22 Dec 2024 12:05:34 +0100 Subject: [PATCH 48/53] implement warn instead of err --- metricbeat/module/pgbouncer/lists/lists.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index bd2f29ed9335..b10962c012f8 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -62,7 +62,8 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for _, s := range results { listValue, ok := s["list"].(string) if !ok { - return fmt.Errorf("expected string type for 'list' but got something else") + m.Logger().Warnf("warning: expected string type for 'list', but got %T", s["list"]) + continue } resultMap[listValue] = s["items"] } From 52ed64d3a1d1c5b6133f0470046676b427c1d5d4 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 22 Dec 2024 12:11:15 +0100 Subject: [PATCH 49/53] add err check --- metricbeat/module/pgbouncer/pools/pools.go | 6 ++++-- metricbeat/module/pgbouncer/stats/stats.go | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/pgbouncer/pools/pools.go b/metricbeat/module/pgbouncer/pools/pools.go index be489de2b342..95bfd6233925 100644 --- a/metricbeat/module/pgbouncer/pools/pools.go +++ b/metricbeat/module/pgbouncer/pools/pools.go @@ -59,8 +59,10 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for _, result := range results { var data mapstr.M - data, _ = schema.Apply(result) - + data, err := schema.Apply(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } reporter.Event(mb.Event{ MetricSetFields: data, }) diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go index bad9514622d6..56cded458622 100644 --- a/metricbeat/module/pgbouncer/stats/stats.go +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -59,8 +59,10 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for _, result := range results { var data mapstr.M - data, _ = schema.Apply(result) - + data, err = schema.Apply(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } reporter.Event(mb.Event{ MetricSetFields: data, }) From b34251289c128dee58bbe2c1e38d82794911bac7 Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 22 Dec 2024 13:28:04 +0100 Subject: [PATCH 50/53] refactor data structures --- metricbeat/docs/fields.asciidoc | 60 +++++++++---------- metricbeat/module/pgbouncer/fields.go | 2 +- .../module/pgbouncer/lists/_meta/fields.yml | 6 +- metricbeat/module/pgbouncer/lists/data.go | 8 ++- .../pgbouncer/lists/lists_integration_test.go | 1 + .../module/pgbouncer/pools/_meta/fields.yml | 22 +++---- metricbeat/module/pgbouncer/pools/data.go | 34 ++++++----- .../pgbouncer/pools/pools_integration_test.go | 6 +- .../module/pgbouncer/stats/_meta/fields.yml | 32 +++++----- metricbeat/module/pgbouncer/stats/data.go | 50 ++++++++++------ .../pgbouncer/stats/stats_integration_test.go | 11 ++-- 11 files changed, 128 insertions(+), 104 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f750552fb80a..1158f0dcbba4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -57816,7 +57816,7 @@ type: long -- -*`pgbouncer.lists.dns_names`*:: +*`pgbouncer.lists.dns.names`*:: + -- Count of DNS names in the cache. @@ -57826,7 +57826,7 @@ type: long -- -*`pgbouncer.lists.dns_zones`*:: +*`pgbouncer.lists.dns.zones`*:: + -- Count of DNS zones in the cache. @@ -57836,7 +57836,7 @@ type: long -- -*`pgbouncer.lists.dns_queries`*:: +*`pgbouncer.lists.dns.queries`*:: + -- Count of in-flight DNS queries. @@ -58320,7 +58320,7 @@ type: keyword -- -*`pgbouncer.pools.cl_active`*:: +*`pgbouncer.pools.client.active`*:: + -- Client connections that are either linked to server connections or are idle with no queries waiting to be processed. @@ -58330,7 +58330,7 @@ type: long -- -*`pgbouncer.pools.cl_waiting`*:: +*`pgbouncer.pools.client.waiting`*:: + -- Client connections that have sent queries but have not yet got a server connection. @@ -58340,7 +58340,7 @@ type: long -- -*`pgbouncer.pools.cl_active_cancel_req`*:: +*`pgbouncer.pools.client.active_cancel_req`*:: + -- Client connections that have forwarded query cancellations to the server and are waiting for the server response. @@ -58350,7 +58350,7 @@ type: long -- -*`pgbouncer.pools.cl_waiting_cancel_req`*:: +*`pgbouncer.pools.client.waiting_cancel_req`*:: + -- Client connections that have not forwarded query cancellations to the server yet. @@ -58360,7 +58360,7 @@ type: long -- -*`pgbouncer.pools.sv_active`*:: +*`pgbouncer.pools.server.active`*:: + -- Server connections that are linked to a client. @@ -58370,7 +58370,7 @@ type: long -- -*`pgbouncer.pools.sv_active_cancel`*:: +*`pgbouncer.pools.server.active_cancel`*:: + -- Server connections that are currently forwarding a cancel request. @@ -58380,7 +58380,7 @@ type: long -- -*`pgbouncer.pools.sv_being_canceled`*:: +*`pgbouncer.pools.server.being_canceled`*:: + -- Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. @@ -58390,7 +58390,7 @@ type: long -- -*`pgbouncer.pools.sv_idle`*:: +*`pgbouncer.pools.server.idle`*:: + -- Server connections that are unused and immediately usable for client queries. @@ -58400,7 +58400,7 @@ type: long -- -*`pgbouncer.pools.sv_used`*:: +*`pgbouncer.pools.server.used`*:: + -- Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. @@ -58410,7 +58410,7 @@ type: long -- -*`pgbouncer.pools.sv_tested`*:: +*`pgbouncer.pools.server.tested`*:: + -- Server connections that are currently running either server_reset_query or server_check_query. @@ -58420,7 +58420,7 @@ type: long -- -*`pgbouncer.pools.sv_login`*:: +*`pgbouncer.pools.server.login`*:: + -- Server connections currently in the process of logging in. @@ -58467,7 +58467,7 @@ type: keyword -- -*`pgbouncer.stats.total_query_count`*:: +*`pgbouncer.stats.query_count.total`*:: + -- Total number of SQL commands pooled by pgbouncer. @@ -58477,7 +58477,7 @@ type: long -- -*`pgbouncer.stats.total_server_assignment_count`*:: +*`pgbouncer.stats.server_assignment_count.total`*:: + -- Total times a server was assigned to a client. @@ -58487,7 +58487,7 @@ type: long -- -*`pgbouncer.stats.total_received`*:: +*`pgbouncer.stats.received.total`*:: + -- Total volume in bytes of network traffic received by pgbouncer. @@ -58497,7 +58497,7 @@ type: long -- -*`pgbouncer.stats.total_sent`*:: +*`pgbouncer.stats.sent.total`*:: + -- Total volume in bytes of network traffic sent by pgbouncer. @@ -58507,7 +58507,7 @@ type: long -- -*`pgbouncer.stats.total_xact_time`*:: +*`pgbouncer.stats.xact_time.total`*:: + -- Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. @@ -58517,7 +58517,7 @@ type: long -- -*`pgbouncer.stats.total_query_time`*:: +*`pgbouncer.stats.query_time.total`*:: + -- Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. @@ -58527,7 +58527,7 @@ type: long -- -*`pgbouncer.stats.total_wait_time`*:: +*`pgbouncer.stats.wait_time.total`*:: + -- Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. @@ -58537,7 +58537,7 @@ type: long -- -*`pgbouncer.stats.total_xact_count`*:: +*`pgbouncer.stats.xact_count.total`*:: + -- Total number of SQL transactions pooled by pgbouncer. @@ -58547,7 +58547,7 @@ type: long -- -*`pgbouncer.stats.avg_xact_count`*:: +*`pgbouncer.stats.xact_count.avg`*:: + -- Average transactions per second in last stat period. @@ -58557,7 +58557,7 @@ type: long -- -*`pgbouncer.stats.avg_query_count`*:: +*`pgbouncer.stats.query_count.avg`*:: + -- Average queries per second in last stat period. @@ -58567,7 +58567,7 @@ type: long -- -*`pgbouncer.stats.avg_server_assignment_count`*:: +*`pgbouncer.stats.server_assignment_count.avg`*:: + -- Average number of times a server as assigned to a client per second in the last stat period. @@ -58577,7 +58577,7 @@ type: long -- -*`pgbouncer.stats.avg_recv`*:: +*`pgbouncer.stats.recv.avg`*:: + -- Average received (from clients) bytes per second. @@ -58587,7 +58587,7 @@ type: long -- -*`pgbouncer.stats.avg_sent`*:: +*`pgbouncer.stats.sent.avg`*:: + -- Average sent (to clients) bytes per second. @@ -58597,7 +58597,7 @@ type: long -- -*`pgbouncer.stats.avg_xact_time`*:: +*`pgbouncer.stats.xact_time.avg`*:: + -- Average transaction duration, in microseconds. @@ -58607,7 +58607,7 @@ type: long -- -*`pgbouncer.stats.avg_query_time`*:: +*`pgbouncer.stats.query_time.avg`*:: + -- Average query duration, in microseconds. @@ -58617,7 +58617,7 @@ type: long -- -*`pgbouncer.stats.avg_wait_time`*:: +*`pgbouncer.stats.wait_time.avg`*:: + -- Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go index e693182bb5d0..9603c42e31e8 100644 --- a/metricbeat/module/pgbouncer/fields.go +++ b/metricbeat/module/pgbouncer/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPgbouncer returns asset data. // This is the base64 encoded zlib format compressed contents of module/pgbouncer. func AssetPgbouncer() string { - return "eJzUm81u4zgSx+95isKceoB0HiCHBfYD2MvOoHfTezYoqiwToVgaknLifvpFkfqKY9mSm6Z3fAokmfUr8l8flJmv8IqHZ2iqgloj0T4AeOU1PsMv3/75t3jtlweAEp20qvGKzDP85QEA4Df0VkkHkrRG6bGEraV6HOrpAcDtyPqNJLNV1TNshXb4AGBRo3D4DJV4ANgq1KV7DmN+BSNq/MjDH39o+HFLbdNdOQHEnwEa6oj31N2aWpla0sp5N1w9ZemMNf687OjNwV5YRa0DZTxaIzQosyVbC/5GtPE0+dbHCeg/x4hTzFJ4UQiH7sPdHleTqY5unCHmz9+pNR5oOw78dNJu69AmtxkGPW2vIdLJ7YVBT9uTWqHx7mlrEVOb5TEHA/B9hw5B2OES+J3w4UKpnCRjQhA9QtF6+Fb1Mn5FbPhRhBprsgcQllpTxi+/CQdCa5IihB9ZfrAGR6A8SGGgQLDYuu7mtvXt1D6B2JMq+yEUmQuzxCPdQAzlYOCsdU2VMqnN95OhDITxwXnh8TSIQ7tn5d5MLL2BiVi6S3cTy2B/sVh6J24mlt7A6URp3Ib/Sp5E/vH7SzARpMJTLIXczQiFIX6QuQ1EGHghxB8tWpUeQ5mvW62qnQ9AnZGnT7W1xvrnK2vwsNfztKqySPuyO4ZAePwniy3Xp02cWad+pAr17zsEHo4nUIBTptIITpPv15LNnlvQCVbC2Pq9rQu0Y3Bp8u4KooQ5cSQSe6G0KLp5ugarxtqTFzrVGvJYUBw8ujhdxWERkbRYovFKaHcHYU2sr4PMJLOr+XKK7mrITBJcyFcWdxBgWSxCyiS3hTQ5xbUQKZOUztM0eJfqyGYXYmXS0QqinFpagZVJT4uIeJN+L1mx7TWAOQW2li271NYC5hTdRbY7iW45Vi6pLSfKKrDlWLlkdYGIWu+8MKUy1cbiHy06fweBTSigo7gOOpP8kvHmFGcy6EzSXckbX7PdQbzR8GK0TBJdSZVTiCvRMsntMpWiot3eQWDB7lKwTPJaxZRTXKvAMknrItNe2I1Wdym9e2HDD74r8DJJ7AqynEK7Ai+T3JaQdRm2sdgIi+Um/IxYo7mHBLvM27PAwPIzDuQts2nZ71CM0zqQt2RfZp87rnHlb3zh/XVrLVsKNnl+w8X4S7ci8/kQx/WHaE7O4ise3sgeS/ySGkQ9sPajz/8mdRu7PPLcwYmNkF7tkx1WCGclJosyOZaAyu+Q06R5xRI8De3f5GGy4VFVaoQ35XdgqP/xGN6E8rwx8QQFQmNJonNYzjrWPX9jz3Ziz1Fh/MBZtN1VQx4O6KEiz1n42NsLK7KRwkjUvBHM4cKW7JuwJZbBjwNE61p0z9E0+oUpwzL1K9KdFOlvW3QNmTmZjyuT2UNejjVeHtDPpOJ92qB5+RwHQ9CM0SK6g0gXmLpJzYDWpWN96GeVpSC6Ke1fH8zSFjgqIFnb8DI9CmXI1kLrA0hqdQkFSqq71MIhOhWwJygJHEFrvNIgtJ4cYPnokItiklQ3Gj32J6fQdlnAU/8F0WmMuOor18lqdkIYLMOqtSaUcw5hVddYKuFRH6B1oU/hSI4y+3xm54g3Ybc3xxtmukA0cdUYriaLfNMMLyx2KF83JWpxeOQV9Ds8gMHhEFj3QFwKT2BbE1cEayhwG4fDQ3+8LU5OJdRMenb7jUfnb+76xwCzrTGs066Edp5ZdOg7z8ie8HfWhZTnJE94MIL3L+ljseZ+RFNVsStzE1yLd47KTZvqSNpvSlpyKMmU0Ajr+6aoFu+qbusxB6gan+A/2PC0xsO3CKGlDo+E++xPPY535oTypqYyYRf5vftdgUF5ZAZp3Yl2m3vjBO02D6OcV9Lx7lHih1V0Xlj/Z+izY94thHxFTncOhmOx4On02oUFj9GzkdQan3Q/ZYat4cu//8U1pBamdGFl4xbrw79GzNF1gS6cU5WJW8DkpKx2N3at4WRwsLekGYmYFiWqfbJMGbn2pNsYh3FvSlsw6N/IvoK3YrtVEnq7K+Yz8eQtgAy9wlLAdyH9hlfkRmKcpjRwzTEavO3QfIgd+EbOVxZZxcqAYLeMEyH9P/ZVKtRsLgDjPS5T+I6yDRn3bIcxDcR7ux4769BJnpyDx3VOhfqW0ieuTAN7/+8K081ZH8ePnwoY/Lcpw2n/6Gjf/E3eqqhJ5IshmV7axk6Ee+s0OtHXilQq9lV6vL/u0YoKj5BCxxb6D2VACxffYPF1RTOvL5gufRHq8fo3FdeS3bYA9ZTjKh8Vo5ladOQONwLLXbIo94n5hzr0JfzXYxeXv3Y1YYQ9N8/JJzXUnS+8SV2Pk7oMnYgWKFsrYhVZ1GqPcXIDrrizuoLo/yjBwxfRedP1xsNexk02+ydTfNnasC86evPuNjGWfn16+F8AAAD//5GzwzU=" + return "eJzUm0tv4zgSx+/5FIU59QBpf4AcFtgHsJedQe+m92yUqLJNhGJpSMpp9adfFPW044fkyPSOT4Eks35F/utBmfkKb1S/QLnNuLKK3BNA0MHQC/zy7Z9/a6798gSQk1dOl0GzfYG/PAEA/EbBaeVBsTGkAuWwcVwMQ62eAPyOXVgrthu9fYENGk9PAI4MoacX2OITwEaTyf1LHPMrWCzokEc+oS7lccdV2V45ASSfHhqKBm/V3hpbGVsy2gffXz1l6YI1+bzu+N3DHp3myoO2gZxFA9pu2BUo32hsrEbfOpyA7nOMOMbMMWCGnvzB3Q7XsN0e3bhALJ+/c2UD8GYYeHXSbuXJLW4zDnraXslsFrcXBz1tTxlNNvjVxhEtbVbG7A3A9x15AnT9JQg7DPFCrr1ia2MQPUNWBfi27WT8RlTKowQFFexqQMeVzZsvv6MHNIYVxvBjJw8W4Bl0AIUWMgJHlW9vbqpQje0z4J513g2h2V6ZJRnpDmLIewMXrRvearu0+W4ytIU4PviAgU6DeHJ7Ue7dxNIZGImlvfQwsfT2J4ulc+JuYukMnE6U1q/kr8WTyD9+f40molRkihWq3RmhCMRPtveBiANPhPijIqeXx9D268bo7S5EoNbI6kNtLaj4fGWNHnZ6HldVEWlXdocQiI9/sthKfVo3M+v1z6VC/fuOQIaTCUTw2m4NgTccurUUs5cWdIS1YGz9XhUZuSG4DAd/A9GCOXEgwj1qg1k7T7dgFVQEDmiWWkMZC7I6kG+mK6snESlHOdmg0fgHCGtkfR5kIpndzJdSdDdDJpLgRL48e4AA82wSUiK5TaRJKa6JSImkdJmmpIdURzE7ESuRjmYQpdTSDKxEeppEJJv0R8lKbM8BTCmwuWzJpTYXMKXorrI9SHTTsVJJbTpRUoFNx0olqytEXAUf0ObabteO/qjIhwcIbEQBLcVt0InktxhvSnEuBp1IujN5m9dsDxBvY3gyWiKJzqRKKcSZaInkdp1Kc1ZtHiCwaHcqWCJ5zWJKKa5ZYImkdZVpj25t9ENK7x5d/MF3Bl4iid1AllJoN+AlktsUsjbDlo5KdJSv48+IBdlHSLDNvB0L9CyfcSBtmV2W/QHFeFkH0pbs6+znjmvc+BtffH9dOSeWok2Z33ix+aVbs/14iOP2QzQnZ/GN6nd2xxK/pgYsetZu9PO/Sd3Hrox86eDEClXQ+8UOLMQxRwszOppAOuxIUqV9oxwC9y3g6GF28VGdG4J3HXZgufsBGd5RB9mcBIaMoHSsyHvKLzrXfufO3u1wL9FhQ8+aVe1VywFqCrDlINn42OMJK7NWaBUZ2RSmcGPD7h1dTnn0pYbGusH2OR5nArR5XK5uZdpTI91tR75ke07yhyuU2EtZljme1hQupeZlg+j1Y1z0QTRED3YzeJ2rndwEeG2aNnU3uyILbKe2e61wkTijQQ2LtRSv42NSll2BxtSguDI5ZKS4aFOOhO1Y0IEhZ/AMlQ3aABozOtxy6JRvhKW4KA0F6k5VkWszQ+DuC9jqjaUj0L5z/NKkCFyC1atsLPcS1rooKNcYyNRQ+djHSHQ3kvt4pucE84Id4TnmOOMZkW1WTwALdiQ3bf9SY0fqbZ2TwfpZVjLsqAZL/UGx9oFmSQKDq2yzMlRARptmOKq7I3DNBG1Rn0ndrfOBfLi7+4cB5yprRbNtmW29c+QptN6xO+HzRTeWPFN5wosBvnuh3xR16V0Mb7fizrmJLvCHROm6Wur42m9aOfak2OZQogtdA1XgD11UxZATdEEr+A+VMrXNQV2C2H7HR+J98acYxrtwmnldcL5gx/m9/Q1CQGVkAan8idZc+ugFWnMZRvuglZedpqKDVfQBXfgz9ORNHs5QvZGkPg/9EVoIfHrtYuysFVc2rJbfe9l+G/n6739JTSnQ5j6ubLMdO/g3io90bZij93prm43inUhF7X7obOMp4mh1SpPiSJHeU34Hrj2bqonDZh/LG7AU3tm9QXC42WjVW580n3eZvAmQsXe4DvgDVVjHzHRPMY5TGvjyGA3ed2QPYge+sQ9bR6JibQHFLesxpv/nrlLF2i0FYLgnpYp+kKpixr3YbTSB+P/hetNtx87y5Bw8T3UqVrblfZLK1LN3/9ow3rx1cfz8oYDBf8s8/mdA42jXCI7ewOhR5GOfTK9tdaNw06TRkb5mpNIRIO6XepHw1z053NIRUuzaYv+hLRj0zdsuua75zGuOcRlaHq97m3EL2bkStDzlsMpHxehMLTpyRxqBaS45Uvs78Pd16Ev8D8k2Ln9ta8IAe6E0LQ8V684X2bTOwxkK0V2jBfLKYVNFJrXaoypxnzCp5xINKX45oE8kePiCrTdtb9zvZfxo438yxeeVi/uio7f0ft3E0q+rp/8FAAD//59Mz/k=" } diff --git a/metricbeat/module/pgbouncer/lists/_meta/fields.yml b/metricbeat/module/pgbouncer/lists/_meta/fields.yml index ada09b863265..bacb75aa7350 100644 --- a/metricbeat/module/pgbouncer/lists/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/lists/_meta/fields.yml @@ -36,15 +36,15 @@ type: long description: > Count of used servers. - - name: dns_names + - name: dns.names type: long description: > Count of DNS names in the cache. - - name: dns_zones + - name: dns.zones type: long description: > Count of DNS zones in the cache. - - name: dns_queries + - name: dns.queries type: long description: > Count of in-flight DNS queries. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go index 6291f882606b..f4b6da2b8dd1 100644 --- a/metricbeat/module/pgbouncer/lists/data.go +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -37,7 +37,9 @@ var schema = s.Schema{ "free": c.Int("free_servers"), "used": c.Int("used_servers"), }, - "dns_names": c.Int("dns_names"), - "dns_zones": c.Int("dns_zones"), - "dns_queries": c.Int("dns_queries"), + "dns": s.Object{ + "names": c.Int("dns_names"), + "zones": c.Int("dns_zones"), + "queries": c.Int("dns_queries"), + }, } diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go index c2fc18093259..73e897d8f7c8 100644 --- a/metricbeat/module/pgbouncer/lists/lists_integration_test.go +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -47,6 +47,7 @@ func TestMetricSet_Fetch(t *testing.T) { assert.Contains(t, event["clients"], "used") assert.Contains(t, event["servers"], "free") assert.Contains(t, event["servers"], "used") + assert.Contains(t, event["dns"], "names") } func getConfig(host string) map[string]interface{} { return map[string]interface{}{ diff --git a/metricbeat/module/pgbouncer/pools/_meta/fields.yml b/metricbeat/module/pgbouncer/pools/_meta/fields.yml index 7396b37872ac..426ad0336032 100644 --- a/metricbeat/module/pgbouncer/pools/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/pools/_meta/fields.yml @@ -12,47 +12,47 @@ type: keyword description: > Name of the user. - - name: cl_active + - name: client.active type: long description: > Client connections that are either linked to server connections or are idle with no queries waiting to be processed. - - name: cl_waiting + - name: client.waiting type: long description: > Client connections that have sent queries but have not yet got a server connection. - - name: cl_active_cancel_req + - name: client.active_cancel_req type: long description: > Client connections that have forwarded query cancellations to the server and are waiting for the server response. - - name: cl_waiting_cancel_req + - name: client.waiting_cancel_req type: long description: > Client connections that have not forwarded query cancellations to the server yet. - - name: sv_active + - name: server.active type: long description: > Server connections that are linked to a client. - - name: sv_active_cancel + - name: server.active_cancel type: long description: > Server connections that are currently forwarding a cancel request. - - name: sv_being_canceled + - name: server.being_canceled type: long description: > Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. - - name: sv_idle + - name: server.idle type: long description: > Server connections that are unused and immediately usable for client queries. - - name: sv_used + - name: server.used type: long description: > Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. - - name: sv_tested + - name: server.tested type: long description: > Server connections that are currently running either server_reset_query or server_check_query. - - name: sv_login + - name: server.login type: long description: > Server connections currently in the process of logging in. diff --git a/metricbeat/module/pgbouncer/pools/data.go b/metricbeat/module/pgbouncer/pools/data.go index f769bc85cc08..2aceaab9215a 100644 --- a/metricbeat/module/pgbouncer/pools/data.go +++ b/metricbeat/module/pgbouncer/pools/data.go @@ -24,19 +24,23 @@ import ( // Based on pgbouncer show pools; var schema = s.Schema{ - "database": c.Str("database"), - "user": c.Str("user"), - "cl_active": c.Int("cl_active"), - "cl_waiting": c.Int("cl_waiting"), - "cl_active_cancel_req": c.Int("cl_active_cancel_req"), - "cl_waiting_cancel_req": c.Int("cl_waiting_cancel_req"), - "sv_active": c.Int("sv_active"), - "sv_active_cancel": c.Int("sv_active_cancel"), - "sv_being_canceled": c.Int("sv_being_canceled"), - "sv_idle": c.Int("sv_idle"), - "sv_used": c.Int("sv_used"), - "sv_tested": c.Int("sv_tested"), - "sv_login": c.Int("sv_login"), - "maxwait_us": c.Int("maxwait_us"), - "pool_mode": c.Str("pool_mode"), + "database": c.Str("database"), + "user": c.Str("user"), + "client": s.Object{ + "active": c.Int("cl_active"), + "waiting": c.Int("cl_waiting"), + "active_cancel_req": c.Int("cl_active_cancel_req"), + "waiting_cancel_req": c.Int("cl_waiting_cancel_req"), + }, + "server": s.Object{ + "active": c.Int("sv_active"), + "active_cancel": c.Int("sv_active_cancel"), + "being_canceled": c.Int("sv_being_canceled"), + "idle": c.Int("sv_idle"), + "used": c.Int("sv_used"), + "tested": c.Int("sv_tested"), + "login": c.Int("sv_login"), + }, + "maxwait_us": c.Int("maxwait_us"), + "pool_mode": c.Str("pool_mode"), } diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go index 255b79f0c21c..08997084ae42 100644 --- a/metricbeat/module/pgbouncer/pools/pools_integration_test.go +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -40,9 +40,9 @@ func TestMetricSet_Fetch(t *testing.T) { require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields assert.Contains(t, event, "user") - assert.Contains(t, event, "cl_active") - assert.Contains(t, event, "cl_active_cancel_req") - assert.Contains(t, event, "sv_idle") + assert.Contains(t, event["client"], "active") + assert.Contains(t, event["client"], "active_cancel_req") + assert.Contains(t, event["server"], "idle") assert.Contains(t, event, "maxwait_us") assert.Contains(t, event, "pool_mode") } diff --git a/metricbeat/module/pgbouncer/stats/_meta/fields.yml b/metricbeat/module/pgbouncer/stats/_meta/fields.yml index 94fc307392fb..f363031b4a50 100644 --- a/metricbeat/module/pgbouncer/stats/_meta/fields.yml +++ b/metricbeat/module/pgbouncer/stats/_meta/fields.yml @@ -8,67 +8,67 @@ type: keyword description: > Name of the database this backend is connected to. - - name: total_query_count + - name: query_count.total type: long description: > Total number of SQL commands pooled by pgbouncer. - - name: total_server_assignment_count + - name: server_assignment_count.total type: long description: > Total times a server was assigned to a client. - - name: total_received + - name: received.total type: long description: > Total volume in bytes of network traffic received by pgbouncer. - - name: total_sent + - name: sent.total type: long description: > Total volume in bytes of network traffic sent by pgbouncer. - - name: total_xact_time + - name: xact_time.total type: long description: > Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. - - name: total_query_time + - name: query_time.total type: long description: > Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. - - name: total_wait_time + - name: wait_time.total type: long description: > Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. - - name: total_xact_count + - name: xact_count.total type: long description: > Total number of SQL transactions pooled by pgbouncer. - - name: avg_xact_count + - name: xact_count.avg type: long description: > Average transactions per second in last stat period. - - name: avg_query_count + - name: query_count.avg type: long description: > Average queries per second in last stat period. - - name: avg_server_assignment_count + - name: server_assignment_count.avg type: long description: > Average number of times a server as assigned to a client per second in the last stat period. - - name: avg_recv + - name: recv.avg type: long description: > Average received (from clients) bytes per second. - - name: avg_sent + - name: sent.avg type: long description: > Average sent (to clients) bytes per second. - - name: avg_xact_time + - name: xact_time.avg type: long description: > Average transaction duration, in microseconds. - - name: avg_query_time + - name: query_time.avg type: long description: > Average query duration, in microseconds. - - name: avg_wait_time + - name: wait_time.avg type: long description: > Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go index 4a720a024da7..cbca6a11352e 100644 --- a/metricbeat/module/pgbouncer/stats/data.go +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -24,21 +24,37 @@ import ( // Based on pgbouncer show stats; var schema = s.Schema{ - "database": c.Str("database"), - "total_query_count": c.Int("total_query_count"), - "total_server_assignment_count": c.Int("total_server_assignment_count"), - "total_received": c.Int("total_received"), - "total_sent": c.Int("total_sent"), - "total_xact_time_us": c.Int("total_xact_time"), - "total_query_time_us": c.Int("total_query_time"), - "total_wait_time_us": c.Int("total_wait_time"), - "total_xact_count": c.Int("total_xact_count"), - "avg_xact_count": c.Int("avg_xact_count"), - "avg_query_count": c.Int("avg_query_count"), - "avg_server_assignment_count": c.Int("avg_server_assignment_count"), - "avg_recv": c.Int("avg_recv"), - "avg_sent": c.Int("avg_sent"), - "avg_xact_time_us": c.Int("avg_xact_time"), - "avg_query_time_us": c.Int("avg_query_time"), - "avg_wait_time_us": c.Int("avg_wait_time"), + "database": c.Str("database"), + "query_count": s.Object{ + "total": c.Int("total_query_count"), + "avg": c.Int("avg_query_count"), + }, + "server_assignment_count": s.Object{ + "total": c.Int("total_server_assignment_count"), + "avg": c.Int("avg_server_assignment_count"), + }, + "received": s.Object{ + "total": c.Int("total_received"), + "avg": c.Int("avg_recv"), + }, + "sent": s.Object{ + "total": c.Int("total_sent"), + "avg": c.Int("avg_sent"), + }, + "xact_time_us": s.Object{ + "total": c.Int("total_xact_time"), + "avg": c.Int("avg_xact_time"), + }, + "query_time_us": s.Object{ + "total": c.Int("total_query_time"), + "avg": c.Int("avg_query_time"), + }, + "wait_time_us": s.Object{ + "total": c.Int("total_wait_time"), + "avg": c.Int("avg_wait_time"), + }, + "xact_count": s.Object{ + "total": c.Int("total_xact_count"), + "avg": c.Int("avg_xact_count"), + }, } diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go index cd03ff2a347a..36f5a0fb8f80 100644 --- a/metricbeat/module/pgbouncer/stats/stats_integration_test.go +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -39,12 +39,13 @@ func TestMetricSet_Fetch(t *testing.T) { require.Empty(t, errs, "Expected no errors during fetch") require.NotEmpty(t, events, "Expected to receive at least one event") event := events[0].MetricSetFields - assert.Contains(t, event, "total_xact_count") - assert.Contains(t, event, "total_server_assignment_count") - assert.Contains(t, event, "total_wait_time_us") + assert.Contains(t, event["xact_count"], "total") + assert.Contains(t, event["xact_count"], "avg") + assert.Contains(t, event["server_assignment_count"], "total") + assert.Contains(t, event["wait_time_us"], "total") assert.Contains(t, event, "database") - assert.Contains(t, event, "total_received") - assert.Contains(t, event, "total_query_count") + assert.Contains(t, event["received"], "total") + assert.Contains(t, event["query_count"], "avg") } func getConfig(host string) map[string]interface{} { return map[string]interface{}{ From 15a98ab35419df4ed00227f8e2baa3cba779416b Mon Sep 17 00:00:00 2001 From: manuelsaks Date: Sun, 22 Dec 2024 13:34:16 +0100 Subject: [PATCH 51/53] update structures --- .../module/pgbouncer/lists/_meta/data.json | 20 ++++---- .../module/pgbouncer/pools/_meta/data.json | 26 +++++----- .../module/pgbouncer/stats/_meta/data.json | 48 ++++++++++++------- 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/_meta/data.json b/metricbeat/module/pgbouncer/lists/_meta/data.json index 0a5a1404b8f8..b1d80919bb31 100644 --- a/metricbeat/module/pgbouncer/lists/_meta/data.json +++ b/metricbeat/module/pgbouncer/lists/_meta/data.json @@ -11,23 +11,25 @@ }, "pgbouncer": { "lists": { + "databases": 1, "users": 2, + "peers": 0, + "pools": 1, + "peer_pools": 0, "clients": { - "free": 47, - "used": 3, + "free": 49, + "used": 1, "login": 0 }, - "peers": 0, - "peer_pools": 0, - "dns_queries": 0, - "databases": 1, - "dns_names": 0, "servers": { "free": 0, "used": 0 }, - "dns_zones": 0, - "pools": 1 + "dns": { + "zones": 0, + "queries": 0, + "names": 0 + } } }, "service": { diff --git a/metricbeat/module/pgbouncer/pools/_meta/data.json b/metricbeat/module/pgbouncer/pools/_meta/data.json index 42521a9badca..638923cb3764 100644 --- a/metricbeat/module/pgbouncer/pools/_meta/data.json +++ b/metricbeat/module/pgbouncer/pools/_meta/data.json @@ -11,21 +11,25 @@ }, "pgbouncer": { "pools": { - "sv_tested": 0, - "cl_waiting": 0, - "sv_login": 0, "maxwait_us": 0, "pool_mode": "statement", - "cl_active": 3, "database": "pgbouncer", - "sv_active_cancel": 0, "user": "pgbouncer", - "cl_active_cancel_req": 0, - "sv_being_canceled": 0, - "sv_idle": 0, - "cl_waiting_cancel_req": 0, - "sv_active": 0, - "sv_used": 0 + "client": { + "active": 2, + "waiting": 0, + "active_cancel_req": 0, + "waiting_cancel_req": 0 + }, + "server": { + "used": 0, + "tested": 0, + "login": 0, + "active": 0, + "active_cancel": 0, + "being_canceled": 0, + "idle": 0 + } } }, "service": { diff --git a/metricbeat/module/pgbouncer/stats/_meta/data.json b/metricbeat/module/pgbouncer/stats/_meta/data.json index 561dddc866ed..b298b01af36d 100644 --- a/metricbeat/module/pgbouncer/stats/_meta/data.json +++ b/metricbeat/module/pgbouncer/stats/_meta/data.json @@ -11,23 +11,39 @@ }, "pgbouncer": { "stats": { - "total_server_assignment_count": 0, - "total_sent": 0, - "avg_sent": 0, - "total_wait_time_us": 0, "database": "pgbouncer", - "total_xact_count": 25, - "total_received": 0, - "total_query_count": 25, - "avg_query_count": 0, - "avg_xact_time_us": 0, - "avg_query_time_us": 0, - "total_query_time_us": 0, - "avg_recv": 0, - "total_xact_time_us": 0, - "avg_xact_count": 0, - "avg_wait_time_us": 0, - "avg_server_assignment_count": 0 + "server_assignment_count": { + "total": 0, + "avg": 0 + }, + "received": { + "total": 0, + "avg": 0 + }, + "query_time_us": { + "total": 0, + "avg": 0 + }, + "sent": { + "total": 0, + "avg": 0 + }, + "xact_count": { + "total": 5, + "avg": 0 + }, + "wait_time_us": { + "avg": 0, + "total": 0 + }, + "query_count": { + "avg": 0, + "total": 5 + }, + "xact_time_us": { + "avg": 0, + "total": 0 + } } }, "service": { From 493e427ef132175c72a8db6af400629e76d51a97 Mon Sep 17 00:00:00 2001 From: manuelsaks <37194183+manuelsaks@users.noreply.github.com> Date: Fri, 28 Feb 2025 20:30:01 +0100 Subject: [PATCH 52/53] remove redundant log settings --- metricbeat/module/pgbouncer/lists/lists.go | 4 ---- metricbeat/module/pgbouncer/mem/mem.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go index b10962c012f8..3b038be67640 100644 --- a/metricbeat/module/pgbouncer/lists/lists.go +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -20,8 +20,6 @@ package lists import ( "context" "fmt" - "log" - "os" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" @@ -29,8 +27,6 @@ import ( // init registers the MetricSet with the central registry. func init() { - log.SetOutput(os.Stderr) - log.SetFlags(0) mb.Registry.MustAddMetricSet("pgbouncer", "lists", New, mb.WithHostParser(pgbouncer.ParseURL), mb.DefaultMetricSet(), diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go index b950e7abbacc..c3604752355c 100644 --- a/metricbeat/module/pgbouncer/mem/mem.go +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -20,8 +20,6 @@ package mem import ( "context" "fmt" - "log" - "os" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" @@ -30,8 +28,6 @@ import ( // init registers the MetricSet with the central registry. func init() { - log.SetOutput(os.Stderr) - log.SetFlags(0) mb.Registry.MustAddMetricSet("pgbouncer", "mem", New, mb.WithHostParser(pgbouncer.ParseURL), mb.DefaultMetricSet(), From 146120629f26badd5f1048269c51473808987818 Mon Sep 17 00:00:00 2001 From: manuelsaks <37194183+manuelsaks@users.noreply.github.com> Date: Sat, 1 Mar 2025 12:33:07 +0000 Subject: [PATCH 53/53] Update reference to make pipeline working --- x-pack/metricbeat/metricbeat.reference.yml | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 7290ab3f3f90..b0600420dac1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1433,9 +1433,11 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass -#----------------------- Prometheus Typed Metrics Module ----------------------- +#------------------------------ Prometheus Module ------------------------------ +# Metrics collected from a Prometheus endpoint - module: prometheus period: 10s + metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1444,20 +1446,14 @@ metricbeat.modules: #username: "user" #password: "secret" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1465,24 +1461,12 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Secure settings for the server using TLS/SSL: - #ssl.certificate: "/etc/pki/server/cert.pem" - #ssl.key: "/etc/pki/server/cert.key" - # Count number of metrics present in Elasticsearch document (default: false) #metrics_count: false - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true - - # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns - #types_patterns: - # counter_patterns: [] - # histogram_patterns: [] + # Secure settings for the server using TLS/SSL: + #ssl.certificate: "/etc/pki/server/cert.pem" + #ssl.key: "/etc/pki/server/cert.key" # Metrics that will be collected using a PromQL #- module: prometheus @@ -1510,11 +1494,9 @@ metricbeat.modules: # params: # query: "some_value" -#------------------------------ Prometheus Module ------------------------------ -# Metrics collected from a Prometheus endpoint +#----------------------- Prometheus Typed Metrics Module ----------------------- - module: prometheus period: 10s - metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1523,14 +1505,20 @@ metricbeat.modules: #username: "user" #password: "secret" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1538,13 +1526,25 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + # Metrics that will be collected using a PromQL #- module: prometheus # metricsets: ["query"]