diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5c965f89db31..17a005de2bfe 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -71,6 +71,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -57255,6 +57256,918 @@ type: keyword -- +[[exported-fields-pgbouncer]] +== PGBouncer fields + +Metrics collected from pgbouncer. + + + +[float] +=== pgbouncer + +PGBouncer metrics. + + + +[float] +=== lists + +Shows various internal information lists. + + + +*`pgbouncer.lists.databases`*:: ++ +-- +Count of databases. + + +type: long + +-- + +*`pgbouncer.lists.users`*:: ++ +-- +Count of users. + + +type: long + +-- + +*`pgbouncer.lists.pools`*:: ++ +-- +Count of pools. + + +type: long + +-- + +*`pgbouncer.lists.clients.free`*:: ++ +-- +Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. + + +type: long + +-- + +*`pgbouncer.lists.clients.used`*:: ++ +-- +Count of used clients. + + +type: long + +-- + +*`pgbouncer.lists.clients.login`*:: ++ +-- +Count of clients in login state. + + +type: long + +-- + +*`pgbouncer.lists.servers.free`*:: ++ +-- +Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. + + +type: long + +-- + +*`pgbouncer.lists.servers.used`*:: ++ +-- +Count of used servers. + + +type: long + +-- + +*`pgbouncer.lists.dns.names`*:: ++ +-- +Count of DNS names in the cache. + + +type: long + +-- + +*`pgbouncer.lists.dns.zones`*:: ++ +-- +Count of DNS zones in the cache. + + +type: long + +-- + +*`pgbouncer.lists.dns.queries`*:: ++ +-- +Count of in-flight DNS queries. + + +type: long + +-- + +[float] +=== mem + +Shows cache memory information for various PgBouncer caches. + + + +*`pgbouncer.mem.user_cache.size`*:: ++ +-- +The size of a single slot in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.used`*:: ++ +-- +Number of used slots in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.free`*:: ++ +-- +Number of available slots in the user cache. + + +type: long + +-- + +*`pgbouncer.mem.user_cache.memtotal`*:: ++ +-- +Total bytes used by the user cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.size`*:: ++ +-- +The size of a single slot in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.used`*:: ++ +-- +Number of used slots in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.free`*:: ++ +-- +Number of available slots in the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.credentials_cache.memtotal`*:: ++ +-- +Total bytes used by the credentials cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.size`*:: ++ +-- +The size of a single slot in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.used`*:: ++ +-- +Number of used slots in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.free`*:: ++ +-- +Number of available slots in the db cache. + + +type: long + +-- + +*`pgbouncer.mem.db_cache.memtotal`*:: ++ +-- +Total bytes used by the db cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.size`*:: ++ +-- +The size of a single slot in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.used`*:: ++ +-- +Number of used slots in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.free`*:: ++ +-- +Number of available slots in the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_cache.memtotal`*:: ++ +-- +Total bytes used by the peer cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.size`*:: ++ +-- +The size of a single slot in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.used`*:: ++ +-- +Number of used slots in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.free`*:: ++ +-- +Number of available slots in the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.peer_pool_cache.memtotal`*:: ++ +-- +Total bytes used by the peer pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.size`*:: ++ +-- +The size of a single slot in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.used`*:: ++ +-- +Number of used slots in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.free`*:: ++ +-- +Number of available slots in the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.pool_cache.memtotal`*:: ++ +-- +Total bytes used by the pool cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.size`*:: ++ +-- +The size of a single slot in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.used`*:: ++ +-- +Number of used slots in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.free`*:: ++ +-- +Number of available slots in the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.outstanding_request_cache.memtotal`*:: ++ +-- +Total bytes used by the outstanding request cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.size`*:: ++ +-- +The size of a single slot in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.used`*:: ++ +-- +Number of used slots in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.free`*:: ++ +-- +Number of available slots in the server cache. + + +type: long + +-- + +*`pgbouncer.mem.server_cache.memtotal`*:: ++ +-- +Total bytes used by the server cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.size`*:: ++ +-- +The size of a single slot in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.used`*:: ++ +-- +Number of used slots in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.free`*:: ++ +-- +Number of available slots in the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.iobuf_cache.memtotal`*:: ++ +-- +Total bytes used by the iobuf cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.size`*:: ++ +-- +The size of a single slot in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.used`*:: ++ +-- +Number of used slots in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.free`*:: ++ +-- +Number of available slots in the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.var_list_cache.memtotal`*:: ++ +-- +Total bytes used by the var list cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.size`*:: ++ +-- +The size of a single slot in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.used`*:: ++ +-- +Number of used slots in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.free`*:: ++ +-- +Number of available slots in the server prepared statement cache. + + +type: long + +-- + +*`pgbouncer.mem.server_prepared_statement_cache.memtotal`*:: ++ +-- +Total bytes used by the server prepared statement cache. + + +type: long + +-- + +[float] +=== pools + +Shows the current state of the connection pools. + + + +*`pgbouncer.pools.database`*:: ++ +-- +Name of the database. + + +type: keyword + +-- + +*`pgbouncer.pools.user`*:: ++ +-- +Name of the user. + + +type: keyword + +-- + +*`pgbouncer.pools.client.active`*:: ++ +-- +Client connections that are either linked to server connections or are idle with no queries waiting to be processed. + + +type: long + +-- + +*`pgbouncer.pools.client.waiting`*:: ++ +-- +Client connections that have sent queries but have not yet got a server connection. + + +type: long + +-- + +*`pgbouncer.pools.client.active_cancel_req`*:: ++ +-- +Client connections that have forwarded query cancellations to the server and are waiting for the server response. + + +type: long + +-- + +*`pgbouncer.pools.client.waiting_cancel_req`*:: ++ +-- +Client connections that have not forwarded query cancellations to the server yet. + + +type: long + +-- + +*`pgbouncer.pools.server.active`*:: ++ +-- +Server connections that are linked to a client. + + +type: long + +-- + +*`pgbouncer.pools.server.active_cancel`*:: ++ +-- +Server connections that are currently forwarding a cancel request. + + +type: long + +-- + +*`pgbouncer.pools.server.being_canceled`*:: ++ +-- +Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. + + +type: long + +-- + +*`pgbouncer.pools.server.idle`*:: ++ +-- +Server connections that are unused and immediately usable for client queries. + + +type: long + +-- + +*`pgbouncer.pools.server.used`*:: ++ +-- +Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. + + +type: long + +-- + +*`pgbouncer.pools.server.tested`*:: ++ +-- +Server connections that are currently running either server_reset_query or server_check_query. + + +type: long + +-- + +*`pgbouncer.pools.server.login`*:: ++ +-- +Server connections currently in the process of logging in. + + +type: long + +-- + +*`pgbouncer.pools.maxwait_us`*:: ++ +-- +Microsecond part of the maximum waiting time. Represents the total wait time in microseconds. + + +type: long + +-- + +*`pgbouncer.pools.pool_mode`*:: ++ +-- +The pooling mode in use. + + +type: keyword + +-- + +[float] +=== stats + +Shows statistics since the process start. + + + +*`pgbouncer.stats.database`*:: ++ +-- +Name of the database this backend is connected to. + + +type: keyword + +-- + +*`pgbouncer.stats.query_count.total`*:: ++ +-- +Total number of SQL commands pooled by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.server_assignment_count.total`*:: ++ +-- +Total times a server was assigned to a client. + + +type: long + +-- + +*`pgbouncer.stats.received.total`*:: ++ +-- +Total volume in bytes of network traffic received by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.sent.total`*:: ++ +-- +Total volume in bytes of network traffic sent by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.xact_time.total`*:: ++ +-- +Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. + + +type: long + +-- + +*`pgbouncer.stats.query_time.total`*:: ++ +-- +Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. + + +type: long + +-- + +*`pgbouncer.stats.wait_time.total`*:: ++ +-- +Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. + + +type: long + +-- + +*`pgbouncer.stats.xact_count.total`*:: ++ +-- +Total number of SQL transactions pooled by pgbouncer. + + +type: long + +-- + +*`pgbouncer.stats.xact_count.avg`*:: ++ +-- +Average transactions per second in last stat period. + + +type: long + +-- + +*`pgbouncer.stats.query_count.avg`*:: ++ +-- +Average queries per second in last stat period. + + +type: long + +-- + +*`pgbouncer.stats.server_assignment_count.avg`*:: ++ +-- +Average number of times a server as assigned to a client per second in the last stat period. + + +type: long + +-- + +*`pgbouncer.stats.recv.avg`*:: ++ +-- +Average received (from clients) bytes per second. + + +type: long + +-- + +*`pgbouncer.stats.sent.avg`*:: ++ +-- +Average sent (to clients) bytes per second. + + +type: long + +-- + +*`pgbouncer.stats.xact_time.avg`*:: ++ +-- +Average transaction duration, in microseconds. + + +type: long + +-- + +*`pgbouncer.stats.query_time.avg`*:: ++ +-- +Average query duration, in microseconds. + + +type: long + +-- + +*`pgbouncer.stats.wait_time.avg`*:: ++ +-- +Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). + + +type: long + +-- + [[exported-fields-php_fpm]] == PHP_FPM fields diff --git a/metricbeat/docs/modules/pgbouncer.asciidoc b/metricbeat/docs/modules/pgbouncer.asciidoc new file mode 100644 index 000000000000..89469e23b513 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer.asciidoc @@ -0,0 +1,130 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +:modulename: pgbouncer +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/_meta/docs.asciidoc + + +[[metricbeat-module-pgbouncer]] +== PGBouncer module + +include::{libbeat-dir}/shared/integration-link.asciidoc[] + +:modulename!: + +PgBouncer is a popular, lightweight connection pooler for PostgreSQL. This module allows you to monitor PgBouncer using Metricbeat, gathering metrics that help optimize performance and maintain stability. + +This module periodically fetches metrics from the PgBouncer poolers available at https://www.pgbouncer.org/. + +Default metricsets include: +- `lists`: Provides a list of databases and users currently connected to PgBouncer. +- `mem`: Monitors memory usage details within PgBouncer, helping detect memory leaks or spikes. +- `pools`: Tracks the status of connection pools, including active and idle connections. +- `stats`: Collects comprehensive statistics on database connections and queries processed. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use PostgreSQL URLs of the following format with PgBouncer database name included: + +[source,yaml] +----------------------------------- +[postgres://][user:pass@]host[:port]/pgbouncer[?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer:40001?sslmode=disable", "postgres://otherhost/pgbouncer:40001"] +---------------------------------------------------------------------- + +Usernames and passwords specified in the URL take precedence over those specified in the `username` and `password` configuration options. +For security best practices, consider using environment variables to pass sensitive information securely. + +[source,yaml] +---- +- module: postgresql + metricsets: ["pools"] + hosts: ["postgres://localhost/pgbouncer:6432"] + username: root + password: test +---- + +:edit_url: + +[float] +=== Example configuration + +The PGBouncer module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::pgbouncer/lists.asciidoc[] + +include::pgbouncer/mem.asciidoc[] + +include::pgbouncer/pools.asciidoc[] + +include::pgbouncer/stats.asciidoc[] + +:edit_url!: diff --git a/metricbeat/docs/modules/pgbouncer/lists.asciidoc b/metricbeat/docs/modules/pgbouncer/lists.asciidoc new file mode 100644 index 000000000000..c6fae28e91bc --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/lists.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-lists]] +=== PGBouncer lists metricset + +include::../../../module/pgbouncer/lists/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/lists/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/mem.asciidoc b/metricbeat/docs/modules/pgbouncer/mem.asciidoc new file mode 100644 index 000000000000..4369707c7fb5 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/mem.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-mem]] +=== PGBouncer mem metricset + +include::../../../module/pgbouncer/mem/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/mem/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/pools.asciidoc b/metricbeat/docs/modules/pgbouncer/pools.asciidoc new file mode 100644 index 000000000000..84032c923ddb --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/pools.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-pools]] +=== PGBouncer pools metricset + +include::../../../module/pgbouncer/pools/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/pools/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/pgbouncer/stats.asciidoc b/metricbeat/docs/modules/pgbouncer/stats.asciidoc new file mode 100644 index 000000000000..78783d6b0ad7 --- /dev/null +++ b/metricbeat/docs/modules/pgbouncer/stats.asciidoc @@ -0,0 +1,27 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc + + +[[metricbeat-metricset-pgbouncer-stats]] +=== PGBouncer stats metricset + +include::../../../module/pgbouncer/stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/pgbouncer/stats/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ee8055b298c8..667b85d02ded 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -251,6 +251,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] +|<> |image:./images/icon-no.png[No prebuilt dashboards] | +.4+| .4+| |<> +|<> +|<> +|<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> |<> @@ -386,6 +391,7 @@ include::modules/openai.asciidoc[] include::modules/openmetrics.asciidoc[] include::modules/oracle.asciidoc[] include::modules/panw.asciidoc[] +include::modules/pgbouncer.asciidoc[] include::modules/php_fpm.asciidoc[] include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index e461c7f6dfa6..3b022afb92a6 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -134,6 +134,11 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/nginx/stubstatus" _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics" _ "github.com/elastic/beats/v7/metricbeat/module/openmetrics/collector" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/lists" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/mem" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/pools" + _ "github.com/elastic/beats/v7/metricbeat/module/pgbouncer/stats" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm/pool" _ "github.com/elastic/beats/v7/metricbeat/module/php_fpm/process" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0d7ceb34057c..b3ee8bd3a3a2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -869,6 +869,40 @@ metricbeat.modules: include: [] exclude: [] +#------------------------------ PGBouncer Module ------------------------------ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass + #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm metricsets: diff --git a/metricbeat/module/pgbouncer/_meta/Dockerfile b/metricbeat/module/pgbouncer/_meta/Dockerfile new file mode 100644 index 000000000000..e729feb0530c --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/Dockerfile @@ -0,0 +1,5 @@ +FROM edoburu/pgbouncer +COPY ./pgbouncer.ini /etc/pgbouncer/pgbouncer.ini +COPY ./userlist.txt /etc/pgbouncer/userlist.txt +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/pgbouncer", "/etc/pgbouncer/pgbouncer.ini"] \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config.reference.yml b/metricbeat/module/pgbouncer/_meta/config.reference.yml new file mode 100644 index 000000000000..cf62e88df9ef --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/config.reference.yml @@ -0,0 +1,32 @@ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass diff --git a/metricbeat/module/pgbouncer/_meta/config.yml b/metricbeat/module/pgbouncer/_meta/config.yml new file mode 100644 index 000000000000..e48057424bd5 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/config.yml @@ -0,0 +1,11 @@ +metricbeat.modules: +- module: pgbouncer + # metricsets: + # - stats + # - lists + # - pools + # - mem + period: 10s + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + # username: test + # password: password \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/config_local.yml b/metricbeat/module/pgbouncer/_meta/config_local.yml new file mode 100644 index 000000000000..a69d02881b67 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/config_local.yml @@ -0,0 +1,17 @@ +metricbeat.modules: +- module: pgbouncer + metricsets: + - mem + - lists + - stats + - pools + period: 10s + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + username: test + password: password +output.elasticsearch: + hosts: ["http://localhost:9200"] + username: elastic + password: xxxx + indices: + - index: "test" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/_meta/docs.asciidoc new file mode 100644 index 000000000000..b01238849d90 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/docs.asciidoc @@ -0,0 +1,51 @@ +include::{libbeat-dir}/shared/integration-link.asciidoc[] + +:modulename!: + +PgBouncer is a popular, lightweight connection pooler for PostgreSQL. This module allows you to monitor PgBouncer using Metricbeat, gathering metrics that help optimize performance and maintain stability. + +This module periodically fetches metrics from the PgBouncer poolers available at https://www.pgbouncer.org/. + +Default metricsets include: +- `lists`: Provides a list of databases and users currently connected to PgBouncer. +- `mem`: Monitors memory usage details within PgBouncer, helping detect memory leaks or spikes. +- `pools`: Tracks the status of connection pools, including active and idle connections. +- `stats`: Collects comprehensive statistics on database connections and queries processed. + +[float] +=== Module-specific configuration notes + +When configuring the `hosts` option, you must use PostgreSQL URLs of the following format with PgBouncer database name included: + +[source,yaml] +----------------------------------- +[postgres://][user:pass@]host[:port]/pgbouncer[?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: pgbouncer + hosts: ["postgres://localhost/pgbouncer:40001?sslmode=disable", "postgres://otherhost/pgbouncer:40001"] +---------------------------------------------------------------------- + +Usernames and passwords specified in the URL take precedence over those specified in the `username` and `password` configuration options. +For security best practices, consider using environment variables to pass sensitive information securely. + +[source,yaml] +---- +- module: postgresql + metricsets: ["pools"] + hosts: ["postgres://localhost/pgbouncer:6432"] + username: root + password: test +---- \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/fields.yml b/metricbeat/module/pgbouncer/_meta/fields.yml new file mode 100644 index 000000000000..9d2e749a695e --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/fields.yml @@ -0,0 +1,12 @@ +- key: pgbouncer + title: "PGBouncer" + description: > + Metrics collected from pgbouncer. + short_config: false + release: ga + fields: + - name: pgbouncer + type: group + description: > + PGBouncer metrics. + fields: \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/pgbouncer.ini b/metricbeat/module/pgbouncer/_meta/pgbouncer.ini new file mode 100644 index 000000000000..e187236e43d1 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/pgbouncer.ini @@ -0,0 +1,18 @@ +[databases] +* = host=postgresql port=5432 dbname=test + +[users] +test = pool_mode=statement max_user_connections=40 + +[pgbouncer] +listen_addr = 0.0.0.0 +listen_port = 6432 +auth_type = trust +admin_users = test +auth_file = /etc/pgbouncer/userlist.txt +max_client_conn = 8000 +default_pool_size = 400 +server_idle_timeout = 120 + +ignore_startup_parameters = extra_float_digits +log_connections = 0 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/_meta/userlist.txt b/metricbeat/module/pgbouncer/_meta/userlist.txt new file mode 100644 index 000000000000..1ada9b43f073 --- /dev/null +++ b/metricbeat/module/pgbouncer/_meta/userlist.txt @@ -0,0 +1 @@ +"test" "password" \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/doc.go b/metricbeat/module/pgbouncer/doc.go new file mode 100644 index 000000000000..0f2098d2e074 --- /dev/null +++ b/metricbeat/module/pgbouncer/doc.go @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package pgbouncer is a Metricbeat module that contains MetricSets. +*/ +package pgbouncer diff --git a/metricbeat/module/pgbouncer/docker-compose.yml b/metricbeat/module/pgbouncer/docker-compose.yml new file mode 100644 index 000000000000..2d31c13945fa --- /dev/null +++ b/metricbeat/module/pgbouncer/docker-compose.yml @@ -0,0 +1,48 @@ +version: '3.9' + +services: + postgresql: + image: postgres:15 + hostname: postgresql + environment: + - POSTGRES_PASSWORD=password + - POSTGRES_USER=test + - POSTGRES_DB=test + - POSTGRES_HOST_AUTH_METHOD=md5 + - POSTGRES_INITDB_ARGS=--auth=md5 + + pgbouncer: + build: + context: ./_meta + dockerfile: Dockerfile + hostname: pgbouncer + ports: + - 6432:6432 + healthcheck: + test: ['CMD-SHELL', 'psql -h localhost -p 6432 -U test -d pgbouncer -c "SHOW STATS"'] + interval: 15s + timeout: 30s + retries: 5 + start_period: 15s + + elasticsearch: + hostname: elasticsearch + extends: + file: ../../../testing/environments/latest.yml + service: elasticsearch + healthcheck: + test: ["CMD-SHELL", "curl -u admin:testing -s http://localhost:9200/_cat/health?h=status | grep -q green"] + retries: 300 + interval: 1s + ports: + - 9200:9200 + + kibana: + extends: + file: ../../../testing/environments/latest.yml + service: kibana + healthcheck: + test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] + retries: 600 + ports: + - 5601:5601 \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/fields.go b/metricbeat/module/pgbouncer/fields.go new file mode 100644 index 000000000000..9603c42e31e8 --- /dev/null +++ b/metricbeat/module/pgbouncer/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package pgbouncer + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "pgbouncer", asset.ModuleFieldsPri, AssetPgbouncer); err != nil { + panic(err) + } +} + +// AssetPgbouncer returns asset data. +// This is the base64 encoded zlib format compressed contents of module/pgbouncer. +func AssetPgbouncer() string { + return "eJzUm0tv4zgSx+/5FIU59QBpf4AcFtgHsJedQe+m92yUqLJNhGJpSMpp9adfFPW044fkyPSOT4Eks35F/utBmfkKb1S/QLnNuLKK3BNA0MHQC/zy7Z9/a6798gSQk1dOl0GzfYG/PAEA/EbBaeVBsTGkAuWwcVwMQ62eAPyOXVgrthu9fYENGk9PAI4MoacX2OITwEaTyf1LHPMrWCzokEc+oS7lccdV2V45ASSfHhqKBm/V3hpbGVsy2gffXz1l6YI1+bzu+N3DHp3myoO2gZxFA9pu2BUo32hsrEbfOpyA7nOMOMbMMWCGnvzB3Q7XsN0e3bhALJ+/c2UD8GYYeHXSbuXJLW4zDnraXslsFrcXBz1tTxlNNvjVxhEtbVbG7A3A9x15AnT9JQg7DPFCrr1ia2MQPUNWBfi27WT8RlTKowQFFexqQMeVzZsvv6MHNIYVxvBjJw8W4Bl0AIUWMgJHlW9vbqpQje0z4J513g2h2V6ZJRnpDmLIewMXrRvearu0+W4ytIU4PviAgU6DeHJ7Ue7dxNIZGImlvfQwsfT2J4ulc+JuYukMnE6U1q/kr8WTyD9+f40molRkihWq3RmhCMRPtveBiANPhPijIqeXx9D268bo7S5EoNbI6kNtLaj4fGWNHnZ6HldVEWlXdocQiI9/sthKfVo3M+v1z6VC/fuOQIaTCUTw2m4NgTccurUUs5cWdIS1YGz9XhUZuSG4DAd/A9GCOXEgwj1qg1k7T7dgFVQEDmiWWkMZC7I6kG+mK6snESlHOdmg0fgHCGtkfR5kIpndzJdSdDdDJpLgRL48e4AA82wSUiK5TaRJKa6JSImkdJmmpIdURzE7ESuRjmYQpdTSDKxEeppEJJv0R8lKbM8BTCmwuWzJpTYXMKXorrI9SHTTsVJJbTpRUoFNx0olqytEXAUf0ObabteO/qjIhwcIbEQBLcVt0InktxhvSnEuBp1IujN5m9dsDxBvY3gyWiKJzqRKKcSZaInkdp1Kc1ZtHiCwaHcqWCJ5zWJKKa5ZYImkdZVpj25t9ENK7x5d/MF3Bl4iid1AllJoN+AlktsUsjbDlo5KdJSv48+IBdlHSLDNvB0L9CyfcSBtmV2W/QHFeFkH0pbs6+znjmvc+BtffH9dOSeWok2Z33ix+aVbs/14iOP2QzQnZ/GN6nd2xxK/pgYsetZu9PO/Sd3Hrox86eDEClXQ+8UOLMQxRwszOppAOuxIUqV9oxwC9y3g6GF28VGdG4J3HXZgufsBGd5RB9mcBIaMoHSsyHvKLzrXfufO3u1wL9FhQ8+aVe1VywFqCrDlINn42OMJK7NWaBUZ2RSmcGPD7h1dTnn0pYbGusH2OR5nArR5XK5uZdpTI91tR75ke07yhyuU2EtZljme1hQupeZlg+j1Y1z0QTRED3YzeJ2rndwEeG2aNnU3uyILbKe2e61wkTijQQ2LtRSv42NSll2BxtSguDI5ZKS4aFOOhO1Y0IEhZ/AMlQ3aABozOtxy6JRvhKW4KA0F6k5VkWszQ+DuC9jqjaUj0L5z/NKkCFyC1atsLPcS1rooKNcYyNRQ+djHSHQ3kvt4pucE84Id4TnmOOMZkW1WTwALdiQ3bf9SY0fqbZ2TwfpZVjLsqAZL/UGx9oFmSQKDq2yzMlRARptmOKq7I3DNBG1Rn0ndrfOBfLi7+4cB5yprRbNtmW29c+QptN6xO+HzRTeWPFN5wosBvnuh3xR16V0Mb7fizrmJLvCHROm6Wur42m9aOfak2OZQogtdA1XgD11UxZATdEEr+A+VMrXNQV2C2H7HR+J98acYxrtwmnldcL5gx/m9/Q1CQGVkAan8idZc+ugFWnMZRvuglZedpqKDVfQBXfgz9ORNHs5QvZGkPg/9EVoIfHrtYuysFVc2rJbfe9l+G/n6739JTSnQ5j6ubLMdO/g3io90bZij93prm43inUhF7X7obOMp4mh1SpPiSJHeU34Hrj2bqonDZh/LG7AU3tm9QXC42WjVW580n3eZvAmQsXe4DvgDVVjHzHRPMY5TGvjyGA3ed2QPYge+sQ9bR6JibQHFLesxpv/nrlLF2i0FYLgnpYp+kKpixr3YbTSB+P/hetNtx87y5Bw8T3UqVrblfZLK1LN3/9ow3rx1cfz8oYDBf8s8/mdA42jXCI7ewOhR5GOfTK9tdaNw06TRkb5mpNIRIO6XepHw1z053NIRUuzaYv+hLRj0zdsuua75zGuOcRlaHq97m3EL2bkStDzlsMpHxehMLTpyRxqBaS45Uvs78Pd16Ev8D8k2Ln9ta8IAe6E0LQ8V684X2bTOwxkK0V2jBfLKYVNFJrXaoypxnzCp5xINKX45oE8kePiCrTdtb9zvZfxo438yxeeVi/uio7f0ft3E0q+rp/8FAAD//59Mz/k=" +} diff --git a/metricbeat/module/pgbouncer/lists/_meta/data.json b/metricbeat/module/pgbouncer/lists/_meta/data.json new file mode 100644 index 000000000000..b1d80919bb31 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2024-08-04T16:40:51.299Z", + "event": { + "dataset": "pgbouncer.lists", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "lists", + "period": 10000 + }, + "pgbouncer": { + "lists": { + "databases": 1, + "users": 2, + "peers": 0, + "pools": 1, + "peer_pools": 0, + "clients": { + "free": 49, + "used": 1, + "login": 0 + }, + "servers": { + "free": 0, + "used": 0 + }, + "dns": { + "zones": 0, + "queries": 0, + "names": 0 + } + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc new file mode 100644 index 000000000000..c5aa07c744e0 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `lists` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/lists/_meta/fields.yml b/metricbeat/module/pgbouncer/lists/_meta/fields.yml new file mode 100644 index 000000000000..bacb75aa7350 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/_meta/fields.yml @@ -0,0 +1,50 @@ +- name: lists + type: group + description: > + Shows various internal information lists. + release: ga + fields: + - name: databases + type: long + description: > + Count of databases. + - name: users + type: long + description: > + Count of users. + - name: pools + type: long + description: > + Count of pools. + - name: clients.free + type: long + description: > + Count of free clients. These are clients that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future clients to avoid allocations. + - name: clients.used + type: long + description: > + Count of used clients. + - name: clients.login + type: long + description: > + Count of clients in login state. + - name: servers.free + type: long + description: > + Count of free servers. These are servers that are disconnected, but PgBouncer keeps the memory around that was allocated for them so it can be reused for future servers to avoid allocations. + - name: servers.used + type: long + description: > + Count of used servers. + - name: dns.names + type: long + description: > + Count of DNS names in the cache. + - name: dns.zones + type: long + description: > + Count of DNS zones in the cache. + - name: dns.queries + type: long + description: > + Count of in-flight DNS queries. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/lists/data.go b/metricbeat/module/pgbouncer/lists/data.go new file mode 100644 index 000000000000..f4b6da2b8dd1 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/data.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package lists + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +var schema = s.Schema{ + "databases": c.Int("databases"), + "users": c.Int("users"), + "peers": c.Int("peers"), + "pools": c.Int("pools"), + "peer_pools": c.Int("peer_pools"), + "clients": s.Object{ + "free": c.Int("free_clients"), + "used": c.Int("used_clients"), + "login": c.Int("login_clients"), + }, + "servers": s.Object{ + "free": c.Int("free_servers"), + "used": c.Int("used_servers"), + }, + "dns": s.Object{ + "names": c.Int("dns_names"), + "zones": c.Int("dns_zones"), + "queries": c.Int("dns_queries"), + }, +} diff --git a/metricbeat/module/pgbouncer/lists/lists.go b/metricbeat/module/pgbouncer/lists/lists.go new file mode 100644 index 000000000000..3b038be67640 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/lists.go @@ -0,0 +1,75 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package lists + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" +) + +// init registers the MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "lists", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + results, err := m.QueryStats(ctx, "SHOW LISTS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + resultMap := make(map[string]interface{}) + for _, s := range results { + listValue, ok := s["list"].(string) + if !ok { + m.Logger().Warnf("warning: expected string type for 'list', but got %T", s["list"]) + continue + } + resultMap[listValue] = s["items"] + } + data, err := schema.Apply(resultMap) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + return nil +} diff --git a/metricbeat/module/pgbouncer/lists/lists_integration_test.go b/metricbeat/module/pgbouncer/lists/lists_integration_test.go new file mode 100644 index 000000000000..73e897d8f7c8 --- /dev/null +++ b/metricbeat/module/pgbouncer/lists/lists_integration_test.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package lists + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + events, errs := mbtest.ReportingFetchV2Error(f) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") + event := events[0].MetricSetFields + assert.Contains(t, event, "databases") + assert.Contains(t, event, "users") + assert.Contains(t, event, "peers") + assert.Contains(t, event, "pools") + assert.Contains(t, event, "peer_pools") + assert.Contains(t, event["clients"], "used") + assert.Contains(t, event["servers"], "free") + assert.Contains(t, event["servers"], "used") + assert.Contains(t, event["dns"], "names") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"lists"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/mem/_meta/data.json b/metricbeat/module/pgbouncer/mem/_meta/data.json new file mode 100644 index 000000000000..43e2d822d952 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/data.json @@ -0,0 +1,86 @@ +{ + "@timestamp": "2024-08-04T17:38:47.013Z", + "event": { + "dataset": "pgbouncer.mem", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "mem", + "period": 10000 + }, + "pgbouncer": { + "mem": { + "credentials_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "peer_pool_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "iobuf_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "pool_cache": { + "used": 1, + "free": 49, + "memtotal": 30800, + "size": 616 + }, + "outstanding_request_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "user_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "server_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "server_prepared_statement_cache": { + "size": 616, + "used": 1, + "free": 49, + "memtotal": 30800 + }, + "db_cache": { + "free": 49, + "memtotal": 30800, + "size": 616, + "used": 1 + }, + "peer_cache": { + "memtotal": 30800, + "size": 616, + "used": 1, + "free": 49 + }, + "var_list_cache": { + "used": 1, + "free": 49, + "memtotal": 30800, + "size": 616 + } + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc new file mode 100644 index 000000000000..bc9bbc13b2a7 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mem` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/mem/_meta/fields.yml b/metricbeat/module/pgbouncer/mem/_meta/fields.yml new file mode 100644 index 000000000000..cae5423a338c --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/_meta/fields.yml @@ -0,0 +1,182 @@ +- name: mem + type: group + description: > + Shows cache memory information for various PgBouncer caches. + release: ga + fields: + - name: user_cache.size + type: long + description: > + The size of a single slot in the user cache. + - name: user_cache.used + type: long + description: > + Number of used slots in the user cache. + - name: user_cache.free + type: long + description: > + Number of available slots in the user cache. + - name: user_cache.memtotal + type: long + description: > + Total bytes used by the user cache. + - name: credentials_cache.size + type: long + description: > + The size of a single slot in the credentials cache. + - name: credentials_cache.used + type: long + description: > + Number of used slots in the credentials cache. + - name: credentials_cache.free + type: long + description: > + Number of available slots in the credentials cache. + - name: credentials_cache.memtotal + type: long + description: > + Total bytes used by the credentials cache. + - name: db_cache.size + type: long + description: > + The size of a single slot in the db cache. + - name: db_cache.used + type: long + description: > + Number of used slots in the db cache. + - name: db_cache.free + type: long + description: > + Number of available slots in the db cache. + - name: db_cache.memtotal + type: long + description: > + Total bytes used by the db cache. + - name: peer_cache.size + type: long + description: > + The size of a single slot in the peer cache. + - name: peer_cache.used + type: long + description: > + Number of used slots in the peer cache. + - name: peer_cache.free + type: long + description: > + Number of available slots in the peer cache. + - name: peer_cache.memtotal + type: long + description: > + Total bytes used by the peer cache. + - name: peer_pool_cache.size + type: long + description: > + The size of a single slot in the peer pool cache. + - name: peer_pool_cache.used + type: long + description: > + Number of used slots in the peer pool cache. + - name: peer_pool_cache.free + type: long + description: > + Number of available slots in the peer pool cache. + - name: peer_pool_cache.memtotal + type: long + description: > + Total bytes used by the peer pool cache. + - name: pool_cache.size + type: long + description: > + The size of a single slot in the pool cache. + - name: pool_cache.used + type: long + description: > + Number of used slots in the pool cache. + - name: pool_cache.free + type: long + description: > + Number of available slots in the pool cache. + - name: pool_cache.memtotal + type: long + description: > + Total bytes used by the pool cache. + - name: outstanding_request_cache.size + type: long + description: > + The size of a single slot in the outstanding request cache. + - name: outstanding_request_cache.used + type: long + description: > + Number of used slots in the outstanding request cache. + - name: outstanding_request_cache.free + type: long + description: > + Number of available slots in the outstanding request cache. + - name: outstanding_request_cache.memtotal + type: long + description: > + Total bytes used by the outstanding request cache. + - name: server_cache.size + type: long + description: > + The size of a single slot in the server cache. + - name: server_cache.used + type: long + description: > + Number of used slots in the server cache. + - name: server_cache.free + type: long + description: > + Number of available slots in the server cache. + - name: server_cache.memtotal + type: long + description: > + Total bytes used by the server cache. + - name: iobuf_cache.size + type: long + description: > + The size of a single slot in the iobuf cache. + - name: iobuf_cache.used + type: long + description: > + Number of used slots in the iobuf cache. + - name: iobuf_cache.free + type: long + description: > + Number of available slots in the iobuf cache. + - name: iobuf_cache.memtotal + type: long + description: > + Total bytes used by the iobuf cache. + - name: var_list_cache.size + type: long + description: > + The size of a single slot in the var list cache. + - name: var_list_cache.used + type: long + description: > + Number of used slots in the var list cache. + - name: var_list_cache.free + type: long + description: > + Number of available slots in the var list cache. + - name: var_list_cache.memtotal + type: long + description: > + Total bytes used by the var list cache. + - name: server_prepared_statement_cache.size + type: long + description: > + The size of a single slot in the server prepared statement cache. + - name: server_prepared_statement_cache.used + type: long + description: > + Number of used slots in the server prepared statement cache. + - name: server_prepared_statement_cache.free + type: long + description: > + Number of available slots in the server prepared statement cache. + - name: server_prepared_statement_cache.memtotal + type: long + description: > + Total bytes used by the server prepared statement cache. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/mem/data.go b/metricbeat/module/pgbouncer/mem/data.go new file mode 100644 index 000000000000..a719893ac313 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/data.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mem + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +var schema = s.Schema{ + "user_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "credentials_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "db_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "peer_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "peer_pool_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "pool_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "outstanding_request_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "server_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "iobuf_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "var_list_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, + "server_prepared_statement_cache": s.Object{ + "size": c.Int("size"), + "used": c.Int("used"), + "free": c.Int("free"), + "memtotal": c.Int("memtotal"), + }, +} diff --git a/metricbeat/module/pgbouncer/mem/mem.go b/metricbeat/module/pgbouncer/mem/mem.go new file mode 100644 index 000000000000..c3604752355c --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/mem.go @@ -0,0 +1,90 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mem + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// init registers the MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "mem", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + // Execute the "SHOW MEM;" query against the database. + results, err := m.QueryStats(ctx, "SHOW MEM;") + if err != nil { + // Return the error if the query fails. + return fmt.Errorf("error in QueryStats: %w", err) + } + + // Initialize an empty map to store aggregated results. + data := mapstr.M{} + + // Iterate over each result from the query. + for _, result := range results { + // Apply the predefined schema to the result to format it properly. + tmpData, err := schema.Apply(result) + if err != nil { + // Log the error and skip this iteration if schema application fails. + m.Logger().Errorf("Error applying schema: %v", err) + continue + } + + // Aggregate the formatted data into the data map. + for k, v := range tmpData { + data[k] = v + } + } + + // Check if there is any data collected. + if len(data) > 0 { + // Create and report an event with the collected data. + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + } + + // Return nil to indicate successful completion. + return nil +} diff --git a/metricbeat/module/pgbouncer/mem/mem_integration_test.go b/metricbeat/module/pgbouncer/mem/mem_integration_test.go new file mode 100644 index 000000000000..9dabbc765cd8 --- /dev/null +++ b/metricbeat/module/pgbouncer/mem/mem_integration_test.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package mem + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + events, errs := mbtest.ReportingFetchV2Error(f) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") + event := events[0].MetricSetFields + assert.Contains(t, event["user_cache"], "size") + assert.Contains(t, event["user_cache"], "used") + assert.Contains(t, event["user_cache"], "free") + assert.Contains(t, event["user_cache"], "memtotal") + assert.Contains(t, event["credentials_cache"], "size") + assert.Contains(t, event["db_cache"], "used") + assert.Contains(t, event["peer_cache"], "free") + assert.Contains(t, event["iobuf_cache"], "memtotal") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"mem"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/metricset.go b/metricbeat/module/pgbouncer/metricset.go new file mode 100644 index 000000000000..cf9f179d98e0 --- /dev/null +++ b/metricbeat/module/pgbouncer/metricset.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pgbouncer + +import ( + "context" + "database/sql" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/logp" + + // Register pgbouncer database/sql driver + _ "github.com/lib/pq" +) + +type MetricSet struct { + mb.BaseMetricSet + db *sql.DB +} + +// NewMetricSet creates a pgbouncer metricset with a pool of connections +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + return &MetricSet{BaseMetricSet: base}, nil +} + +// DB creates a database connection, it must be freed after use with `Close()` +func (ms *MetricSet) DB(ctx context.Context) (*sql.Conn, error) { + if ms.db == nil { + db, err := sql.Open("postgres", ms.HostData().URI) + if err != nil { + return nil, fmt.Errorf("failed to open connection: %w", err) + } + ms.db = db + } + return ms.db.Conn(ctx) +} + +// QueryStats makes the database call for a given metric +func (ms *MetricSet) QueryStats(ctx context.Context, query string) ([]map[string]interface{}, error) { + db, err := ms.DB(ctx) + if err != nil { + return nil, fmt.Errorf("failed to obtain a connection with the database: %w", err) + } + defer db.Close() + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query database: %w", err) + } + columns, err := rows.Columns() + if err != nil { + return nil, fmt.Errorf("scanning columns: %w", err) + } + vals := make([][]byte, len(columns)) + valPointers := make([]interface{}, len(columns)) + for i := range vals { + valPointers[i] = &vals[i] + } + results := []map[string]interface{}{} + for rows.Next() { + err = rows.Scan(valPointers...) + if err != nil { + return nil, fmt.Errorf("scanning row: %w", err) + } + result := map[string]interface{}{} + for i, col := range columns { + result[col] = string(vals[i]) + } + logp.Debug("postgresql", "Result: %v", result) + results = append(results, result) + } + return results, nil +} + +// Close closes the metricset and its connections +func (ms *MetricSet) Close() error { + if ms.db == nil { + return nil + } + if err := ms.db.Close(); err != nil { + return fmt.Errorf("failed to close connection: %w", err) + } + return nil +} diff --git a/metricbeat/module/pgbouncer/metricset_integration_test.go b/metricbeat/module/pgbouncer/metricset_integration_test.go new file mode 100644 index 000000000000..7a80ac2b93d5 --- /dev/null +++ b/metricbeat/module/pgbouncer/metricset_integration_test.go @@ -0,0 +1,117 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package pgbouncer + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" +) + +func TestNewMetricSet(t *testing.T) { + base := mb.BaseMetricSet{} + metricSet, err := NewMetricSet(base) + assert.NoError(t, err) + assert.NotNil(t, metricSet) +} + +func TestDBConnection(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + defer db.Close() + ctx := context.Background() + metricSet := MetricSet{ + db: db, + } + conn, err := metricSet.DB(ctx) + assert.NoError(t, err) + assert.NotNil(t, conn, "The database connection should not be nil") + if conn != nil { + defer conn.Close() + } +} + +func TestQueryStats(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + + defer db.Close() + metricSet := MetricSet{ + db: db, + } + ctx := context.Background() + query := "SHOW STATS;" + results, err := metricSet.QueryStats(ctx, query) + assert.NoError(t, err) + assert.NotNil(t, results) + assert.NotEmpty(t, results) +} + +func TestClose(t *testing.T) { + db, err := connectDatabase(t) + if err != nil { + t.Fatalf("Error opening database: %s", err) + } + + metricSet := MetricSet{ + db: db, + } + + err = metricSet.Close() + assert.NoError(t, err) + + err = db.Ping() + assert.Error(t, err) +} + +func connectDatabase(t *testing.T) (*sql.DB, error) { + service := compose.EnsureUp(t, "pgbouncer") + config := getConfig(service.Host()) + + dsn := fmt.Sprintf("postgres://%s:%s@%s", + config["username"].(string), + config["password"].(string), + config["hosts"].([]string)[0], + ) + + db, err := sql.Open("postgres", dsn) + return db, err +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"stats"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/pgbouncer.go b/metricbeat/module/pgbouncer/pgbouncer.go new file mode 100644 index 000000000000..f1e907bdfd1e --- /dev/null +++ b/metricbeat/module/pgbouncer/pgbouncer.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package pgbouncer is Metricbeat module for pgbouncer pooler. +*/ +package pgbouncer + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/lib/pq" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +func init() { + // Register the ModuleFactory function for the "pgbouncer" module. + if err := mb.Registry.AddModule("pgbouncer", NewModule); err != nil { + panic(err) + } +} + +// NewModule returns a new instance of the module +func NewModule(base mb.BaseModule) (mb.Module, error) { + // Validate that at least one host has been specified. + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + }{} + if err := base.UnpackConfig(&config); err != nil { + return nil, err + } + + return &base, nil +} + +// ParseURL is the pgbouncer host parser +func ParseURL(mod mb.Module, rawURL string) (mb.HostData, error) { + c := struct { + Username string `config:"username"` + Password string `config:"password"` + }{} + if err := mod.UnpackConfig(&c); err != nil { + return mb.HostData{}, err + } + + if parts := strings.SplitN(rawURL, "://", 2); len(parts) != 2 { + // Add scheme. + rawURL = fmt.Sprintf("postgres://%s", rawURL) + } + + u, err := url.Parse(rawURL) + if err != nil { + return mb.HostData{}, fmt.Errorf("error parsing URL: %w", err) + } + + parse.SetURLUser(u, c.Username, c.Password) + + if timeout := mod.Config().Timeout; timeout > 0 { + q := u.Query() + q.Set("connect_timeout", strconv.Itoa(int(timeout.Seconds()))) + u.RawQuery = q.Encode() + } + + // https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING + connString, err := pq.ParseURL(u.String()) + if err != nil { + return mb.HostData{}, err + } + + h := parse.NewHostDataFromURL(u) + + // Store the connection string instead of URL to avoid the cost of sql.Open + // parsing the URL on each call. + h.URI = connString + + // Postgres URLs can use a host query param to specify the host. This is + // used for unix domain sockets (postgres:///dbname?host=/var/lib/postgres). + if host := u.Query().Get("host"); u.Host == "" && host != "" { + h.Host = host + } + + return h, nil +} diff --git a/metricbeat/module/pgbouncer/pgbouncer_test.go b/metricbeat/module/pgbouncer/pgbouncer_test.go new file mode 100644 index 000000000000..8951348720e9 --- /dev/null +++ b/metricbeat/module/pgbouncer/pgbouncer_test.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pgbouncer + +import ( + "fmt" + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/management/status" + "github.com/elastic/beats/v7/metricbeat/mb" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseUrl(t *testing.T) { + tests := []struct { + Name string + URL string + Username string + Password string + Timeout time.Duration + Expected string + ExpectErr bool + }{ + { + Name: "simple test", + URL: "postgres://host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "no port", + URL: "postgres://host1/pgbouncer", + Expected: "dbname='pgbouncer' host='host1'", + ExpectErr: false, + }, + { + Name: "user/pass in URL", + URL: "postgres://user:pass@host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' password='pass' port='6432' user='user'", + ExpectErr: false, + }, + { + Name: "user/pass in params", + URL: "postgres://host1:6432/pgbouncer", + Username: "user", + Password: "secret", + Expected: "dbname='pgbouncer' host='host1' password='secret' port='6432' user='user'", + ExpectErr: false, + }, + { + Name: "user/pass in URL take precedence", + URL: "postgres://user1:pass@host1:6432/pgbouncer", + Username: "user", + Password: "secret", + Expected: "dbname='pgbouncer' host='host1' password='pass' port='6432' user='user1'", + ExpectErr: false, + }, + { + Name: "timeout no override", + URL: "postgres://host1:6432/pgbouncer?connect_timeout=2", + Expected: "connect_timeout='2' dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "timeout from param", + URL: "postgres://host1:6432/pgbouncer", + Timeout: 3 * time.Second, + Expected: "connect_timeout='3' dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "user/pass in URL take precedence, and timeout override", + URL: "postgres://user1:pass@host1:6432/pgbouncer?connect_timeout=2", + Username: "user", + Password: "secret", + Timeout: 3 * time.Second, + Expected: "connect_timeout='3' dbname='pgbouncer' host='host1' password='pass' port='6432' user='user1'", + ExpectErr: false, + }, + { + Name: "unix socket", + URL: "postgresql:///pgbouncer?host=/var/lib/postgresql", + Expected: "dbname='pgbouncer' host='/var/lib/postgresql'", + ExpectErr: false, + }, + { + Name: "no ssl", + URL: "postgresql://localhost:6432/pgbouncer?sslmode=disable", + Expected: "dbname='pgbouncer' host='localhost' port='6432' sslmode='disable'", + ExpectErr: false, + }, + { + Name: "no scheme", + URL: "host1:6432/pgbouncer", + Expected: "dbname='pgbouncer' host='host1' port='6432'", + ExpectErr: false, + }, + { + Name: "invalid url", + URL: "://pgbouncer:6432", + ExpectErr: true, + }, + { + Name: "invalid schema", + URL: "abcd://", + ExpectErr: true, + }, + } + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + mod := mbtest.NewTestModule(t, map[string]interface{}{ + "username": test.Username, + "password": test.Password, + }) + mod.ModConfig.Timeout = test.Timeout + + hostData, err := ParseURL(mod, test.URL) + + if test.ExpectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.Expected, hostData.URI) + } + }) + } +} + +type MockModule struct { + Username string + Password string + Timeout time.Duration + statusReporter status.StatusReporter + statusUpdates []statusUpdate +} +type statusUpdate struct { + status status.Status + msg string +} + +func (m *MockModule) Name() string { + return "mockmodule" +} +func (m *MockModule) SetStatusReporter(statusReporter status.StatusReporter) { + m.statusReporter = statusReporter +} +func (m *MockModule) UpdateStatus(status status.Status, msg string) { + m.statusUpdates = append(m.statusUpdates, statusUpdate{status: status, msg: msg}) +} +func (m *MockModule) Config() mb.ModuleConfig { + return mb.ModuleConfig{ + Timeout: m.Timeout, + } +} + +func (m *MockModule) UnpackConfig(to interface{}) error { + if m.Username == "" { + return fmt.Errorf("simulated config error") + } + c, ok := to.(*struct { + Username string `config:"username"` + Password string `config:"password"` + }) + if !ok { + return fmt.Errorf("type assertion failed") + } + c.Username = m.Username + c.Password = m.Password + return nil +} + +func TestParseURL_UnpackConfigError(t *testing.T) { + mod := &MockModule{ + Username: "", + Password: "pass", + Timeout: 5 * time.Second, + } + _, err := ParseURL(mod, "postgres://localhost:5432") + assert.Error(t, err) + assert.Contains(t, err.Error(), "simulated config error") +} diff --git a/metricbeat/module/pgbouncer/pools/_meta/data.json b/metricbeat/module/pgbouncer/pools/_meta/data.json new file mode 100644 index 000000000000..638923cb3764 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2024-08-04T16:48:49.028Z", + "event": { + "dataset": "pgbouncer.pools", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "pools", + "period": 10000 + }, + "pgbouncer": { + "pools": { + "maxwait_us": 0, + "pool_mode": "statement", + "database": "pgbouncer", + "user": "pgbouncer", + "client": { + "active": 2, + "waiting": 0, + "active_cancel_req": 0, + "waiting_cancel_req": 0 + }, + "server": { + "used": 0, + "tested": 0, + "login": 0, + "active": 0, + "active_cancel": 0, + "being_canceled": 0, + "idle": 0 + } + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc new file mode 100644 index 000000000000..3bf80cfc02eb --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `pools` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/pools/_meta/fields.yml b/metricbeat/module/pgbouncer/pools/_meta/fields.yml new file mode 100644 index 000000000000..426ad0336032 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/_meta/fields.yml @@ -0,0 +1,66 @@ +- name: pools + type: group + description: > + Shows the current state of the connection pools. + release: ga + fields: + - name: database + type: keyword + description: > + Name of the database. + - name: user + type: keyword + description: > + Name of the user. + - name: client.active + type: long + description: > + Client connections that are either linked to server connections or are idle with no queries waiting to be processed. + - name: client.waiting + type: long + description: > + Client connections that have sent queries but have not yet got a server connection. + - name: client.active_cancel_req + type: long + description: > + Client connections that have forwarded query cancellations to the server and are waiting for the server response. + - name: client.waiting_cancel_req + type: long + description: > + Client connections that have not forwarded query cancellations to the server yet. + - name: server.active + type: long + description: > + Server connections that are linked to a client. + - name: server.active_cancel + type: long + description: > + Server connections that are currently forwarding a cancel request. + - name: server.being_canceled + type: long + description: > + Servers that normally could become idle but are waiting to do so until all in-flight cancel requests have completed that were sent to cancel a query on this server. + - name: server.idle + type: long + description: > + Server connections that are unused and immediately usable for client queries. + - name: server.used + type: long + description: > + Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again. + - name: server.tested + type: long + description: > + Server connections that are currently running either server_reset_query or server_check_query. + - name: server.login + type: long + description: > + Server connections currently in the process of logging in. + - name: maxwait_us + type: long + description: > + Microsecond part of the maximum waiting time. Represents the total wait time in microseconds. + - name: pool_mode + type: keyword + description: > + The pooling mode in use. \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/pools/data.go b/metricbeat/module/pgbouncer/pools/data.go new file mode 100644 index 000000000000..2aceaab9215a --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/data.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pools + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +// Based on pgbouncer show pools; +var schema = s.Schema{ + "database": c.Str("database"), + "user": c.Str("user"), + "client": s.Object{ + "active": c.Int("cl_active"), + "waiting": c.Int("cl_waiting"), + "active_cancel_req": c.Int("cl_active_cancel_req"), + "waiting_cancel_req": c.Int("cl_waiting_cancel_req"), + }, + "server": s.Object{ + "active": c.Int("sv_active"), + "active_cancel": c.Int("sv_active_cancel"), + "being_canceled": c.Int("sv_being_canceled"), + "idle": c.Int("sv_idle"), + "used": c.Int("sv_used"), + "tested": c.Int("sv_tested"), + "login": c.Int("sv_login"), + }, + "maxwait_us": c.Int("maxwait_us"), + "pool_mode": c.Str("pool_mode"), +} diff --git a/metricbeat/module/pgbouncer/pools/pools.go b/metricbeat/module/pgbouncer/pools/pools.go new file mode 100644 index 000000000000..95bfd6233925 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/pools.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pools + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// init registers the MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "pools", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + results, err := m.QueryStats(ctx, "SHOW POOLS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + + for _, result := range results { + var data mapstr.M + + data, err := schema.Apply(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + } + return nil +} diff --git a/metricbeat/module/pgbouncer/pools/pools_integration_test.go b/metricbeat/module/pgbouncer/pools/pools_integration_test.go new file mode 100644 index 000000000000..08997084ae42 --- /dev/null +++ b/metricbeat/module/pgbouncer/pools/pools_integration_test.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package pools + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + events, errs := mbtest.ReportingFetchV2Error(f) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") + event := events[0].MetricSetFields + assert.Contains(t, event, "user") + assert.Contains(t, event["client"], "active") + assert.Contains(t, event["client"], "active_cancel_req") + assert.Contains(t, event["server"], "idle") + assert.Contains(t, event, "maxwait_us") + assert.Contains(t, event, "pool_mode") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"pools"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/module/pgbouncer/stats/_meta/data.json b/metricbeat/module/pgbouncer/stats/_meta/data.json new file mode 100644 index 000000000000..b298b01af36d --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/data.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2024-08-04T16:49:49.268Z", + "event": { + "dataset": "pgbouncer.stats", + "duration": 3184900, + "module": "pgbouncer" + }, + "metricset": { + "name": "stats", + "period": 10000 + }, + "pgbouncer": { + "stats": { + "database": "pgbouncer", + "server_assignment_count": { + "total": 0, + "avg": 0 + }, + "received": { + "total": 0, + "avg": 0 + }, + "query_time_us": { + "total": 0, + "avg": 0 + }, + "sent": { + "total": 0, + "avg": 0 + }, + "xact_count": { + "total": 5, + "avg": 0 + }, + "wait_time_us": { + "avg": 0, + "total": 0 + }, + "query_count": { + "avg": 0, + "total": 5 + }, + "xact_time_us": { + "avg": 0, + "total": 0 + } + } + }, + "service": { + "address": "postgresql://localhost:6432/pgbouncer?connect_timeout=10&sslmode=disable", + "type": "pgbouncer" + } +} \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc b/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc new file mode 100644 index 000000000000..713f5d3f61e7 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `stats` metricset of the PgBouncer module. diff --git a/metricbeat/module/pgbouncer/stats/_meta/fields.yml b/metricbeat/module/pgbouncer/stats/_meta/fields.yml new file mode 100644 index 000000000000..f363031b4a50 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/_meta/fields.yml @@ -0,0 +1,74 @@ +- name: stats + type: group + description: > + Shows statistics since the process start. + release: ga + fields: + - name: database + type: keyword + description: > + Name of the database this backend is connected to. + - name: query_count.total + type: long + description: > + Total number of SQL commands pooled by pgbouncer. + - name: server_assignment_count.total + type: long + description: > + Total times a server was assigned to a client. + - name: received.total + type: long + description: > + Total volume in bytes of network traffic received by pgbouncer. + - name: sent.total + type: long + description: > + Total volume in bytes of network traffic sent by pgbouncer. + - name: xact_time.total + type: long + description: > + Total number of microseconds spent by pgbouncer when connected to PostgreSQL in a transaction, either idle in transaction or executing queries. + - name: query_time.total + type: long + description: > + Total number of microseconds spent by pgbouncer when actively connected to PostgreSQL, executing queries. + - name: wait_time.total + type: long + description: > + Time spent by clients waiting for a server, in microseconds. Updated when a client connection is assigned a backend connection. + - name: xact_count.total + type: long + description: > + Total number of SQL transactions pooled by pgbouncer. + - name: xact_count.avg + type: long + description: > + Average transactions per second in last stat period. + - name: query_count.avg + type: long + description: > + Average queries per second in last stat period. + - name: server_assignment_count.avg + type: long + description: > + Average number of times a server as assigned to a client per second in the last stat period. + - name: recv.avg + type: long + description: > + Average received (from clients) bytes per second. + - name: sent.avg + type: long + description: > + Average sent (to clients) bytes per second. + - name: xact_time.avg + type: long + description: > + Average transaction duration, in microseconds. + - name: query_time.avg + type: long + description: > + Average query duration, in microseconds. + - name: wait_time.avg + type: long + description: > + Time spent by clients waiting for a server, in microseconds (average of the wait times for clients assigned a backend during the current stats_period). \ No newline at end of file diff --git a/metricbeat/module/pgbouncer/stats/data.go b/metricbeat/module/pgbouncer/stats/data.go new file mode 100644 index 000000000000..cbca6a11352e --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/data.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" +) + +// Based on pgbouncer show stats; +var schema = s.Schema{ + "database": c.Str("database"), + "query_count": s.Object{ + "total": c.Int("total_query_count"), + "avg": c.Int("avg_query_count"), + }, + "server_assignment_count": s.Object{ + "total": c.Int("total_server_assignment_count"), + "avg": c.Int("avg_server_assignment_count"), + }, + "received": s.Object{ + "total": c.Int("total_received"), + "avg": c.Int("avg_recv"), + }, + "sent": s.Object{ + "total": c.Int("total_sent"), + "avg": c.Int("avg_sent"), + }, + "xact_time_us": s.Object{ + "total": c.Int("total_xact_time"), + "avg": c.Int("avg_xact_time"), + }, + "query_time_us": s.Object{ + "total": c.Int("total_query_time"), + "avg": c.Int("avg_query_time"), + }, + "wait_time_us": s.Object{ + "total": c.Int("total_wait_time"), + "avg": c.Int("avg_wait_time"), + }, + "xact_count": s.Object{ + "total": c.Int("total_xact_count"), + "avg": c.Int("avg_xact_count"), + }, +} diff --git a/metricbeat/module/pgbouncer/stats/stats.go b/metricbeat/module/pgbouncer/stats/stats.go new file mode 100644 index 000000000000..56cded458622 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/stats.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/pgbouncer" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// init registers the MetricSet with the central registry. +func init() { + mb.Registry.MustAddMetricSet("pgbouncer", "stats", New, + mb.WithHostParser(pgbouncer.ParseURL), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *pgbouncer.MetricSet +} + +// New creates a new instance of the MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := pgbouncer.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It publishes the event which is then forwarded to the output. In case of an error, an error is reported. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + results, err := m.QueryStats(ctx, "SHOW STATS;") + if err != nil { + return fmt.Errorf("error in QueryStats: %w", err) + } + + for _, result := range results { + var data mapstr.M + + data, err = schema.Apply(result) + if err != nil { + return fmt.Errorf("error mapping result: %w", err) + } + reporter.Event(mb.Event{ + MetricSetFields: data, + }) + } + return nil +} diff --git a/metricbeat/module/pgbouncer/stats/stats_integration_test.go b/metricbeat/module/pgbouncer/stats/stats_integration_test.go new file mode 100644 index 000000000000..36f5a0fb8f80 --- /dev/null +++ b/metricbeat/module/pgbouncer/stats/stats_integration_test.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package stats + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/postgresql" + + "github.com/elastic/beats/v7/libbeat/tests/compose" +) + +func TestMetricSet_Fetch(t *testing.T) { + service := compose.EnsureUp(t, "pgbouncer") + + f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + events, errs := mbtest.ReportingFetchV2Error(f) + require.Empty(t, errs, "Expected no errors during fetch") + require.NotEmpty(t, events, "Expected to receive at least one event") + event := events[0].MetricSetFields + assert.Contains(t, event["xact_count"], "total") + assert.Contains(t, event["xact_count"], "avg") + assert.Contains(t, event["server_assignment_count"], "total") + assert.Contains(t, event["wait_time_us"], "total") + assert.Contains(t, event, "database") + assert.Contains(t, event["received"], "total") + assert.Contains(t, event["query_count"], "avg") +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "pgbouncer", + "metricsets": []string{"stats"}, + "hosts": []string{"localhost:6432/pgbouncer?sslmode=disable"}, + "username": "test", + "password": postgresql.GetEnvPassword(), + } +} diff --git a/metricbeat/modules.d/pgbouncer.yml.disabled b/metricbeat/modules.d/pgbouncer.yml.disabled new file mode 100644 index 000000000000..f96114664cfd --- /dev/null +++ b/metricbeat/modules.d/pgbouncer.yml.disabled @@ -0,0 +1,14 @@ +# Module: pgbouncer +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-pgbouncer.html + +metricbeat.modules: +- module: pgbouncer + # metricsets: + # - stats + # - lists + # - pools + # - mem + period: 10s + hosts: ["postgresql://localhost:6432/pgbouncer?sslmode=disable"] + # username: test + # password: password \ No newline at end of file diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 40aeb4742c17..b0600420dac1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1358,6 +1358,40 @@ metricbeat.modules: hosts: ["localhost"] +#------------------------------ PGBouncer Module ------------------------------ +- module: pgbouncer + enabled: true + metricsets: + # Monitors the current state of the connection pools including metrics like active connections, idle connections, waiting connections, and max allowed connections + # Essential for assessing load and capacity + - pools + + # Provides comprehensive data on connections, pools, and queries including total queries, average query duration, and query rates + # Useful for performance tuning and identifying bottlenecks + - stats + + # Details current activity of the PgBouncer service showing lists of databases and users + # Helps in auditing access and understanding user distribution + - lists + + # Reports on the sizes of internal memory allocations to track memory usage and potential leaks, critical for maintaining system stability + - memory + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost/pgbouncer:6432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + # You have to specify the pgbouncer as the database name. + hosts: ["postgres://localhost/pgbouncer:6432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass + #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm metricsets: @@ -1399,9 +1433,11 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass -#----------------------- Prometheus Typed Metrics Module ----------------------- +#------------------------------ Prometheus Module ------------------------------ +# Metrics collected from a Prometheus endpoint - module: prometheus period: 10s + metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1410,20 +1446,14 @@ metricbeat.modules: #username: "user" #password: "secret" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1431,24 +1461,12 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Secure settings for the server using TLS/SSL: - #ssl.certificate: "/etc/pki/server/cert.pem" - #ssl.key: "/etc/pki/server/cert.key" - # Count number of metrics present in Elasticsearch document (default: false) #metrics_count: false - # Use Elasticsearch histogram type to store histograms (beta, default: false) - # This will change the default layout and put metric type in the field name - #use_types: true - - # Store counter rates instead of original cumulative counters (experimental, default: false) - #rate_counters: true - - # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns - #types_patterns: - # counter_patterns: [] - # histogram_patterns: [] + # Secure settings for the server using TLS/SSL: + #ssl.certificate: "/etc/pki/server/cert.pem" + #ssl.key: "/etc/pki/server/cert.key" # Metrics that will be collected using a PromQL #- module: prometheus @@ -1476,11 +1494,9 @@ metricbeat.modules: # params: # query: "some_value" -#------------------------------ Prometheus Module ------------------------------ -# Metrics collected from a Prometheus endpoint +#----------------------- Prometheus Typed Metrics Module ----------------------- - module: prometheus period: 10s - metricsets: ["collector"] hosts: ["localhost:9090"] metrics_path: /metrics #metrics_filters: @@ -1489,14 +1505,20 @@ metricbeat.modules: #username: "user" #password: "secret" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # This can be used for service account based authorization: #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true # Metrics sent by a Prometheus server using remote_write option #- module: prometheus @@ -1504,13 +1526,25 @@ metricbeat.modules: # host: "localhost" # port: "9201" - # Count number of metrics present in Elasticsearch document (default: false) - #metrics_count: false - # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + # Count number of metrics present in Elasticsearch document (default: false) + #metrics_count: false + + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + # Metrics that will be collected using a PromQL #- module: prometheus # metricsets: ["query"]