From de7d63e194d23922481ef2fa39de8e744358c7f9 Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 09:46:11 +0300 Subject: [PATCH 1/3] added breaking changes feature Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 34 ++++++++++++++++ cmd/postgres_exporter/postgres_exporter.go | 39 +++++++++++++++---- .../postgres_exporter_test.go | 2 +- queries.yaml | 18 ++++++++- 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 cmd/postgres_exporter/pg_breaking_changes.go diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go new file mode 100644 index 000000000..ddd4a76aa --- /dev/null +++ b/cmd/postgres_exporter/pg_breaking_changes.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/blang/semver" +) + +type BreakingChanges struct { + Version string `yaml:"version"` + ver semver.Version + Columns map[string]string `yaml:"columns"` +} + +func (bc *BreakingChanges) parseVerTolerant() error { + bcVer, err := semver.ParseTolerant(bc.Version) + if err != nil { + return err + } + + bc.ver = bcVer + return nil +} diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 36c524c8e..132e423a7 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,11 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + BreakingChanges []BreakingChanges `yaml:"breakingChanges"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint @@ -517,7 +518,7 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][] return resultMap } -func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[string]string, error) { +func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]intermediateMetricMap, map[string]string, error) { var userQueries UserQueries err := yaml.Unmarshal(content, &userQueries) @@ -532,6 +533,30 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query + columnT := make(map[string]string) + for i := range specs.BreakingChanges { + if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { + return nil, nil, err + } + + if pgVersion.GE(specs.BreakingChanges[i].ver) { + for t := range specs.BreakingChanges[i].Columns { + columnT[t] = specs.BreakingChanges[i].Columns[t] + } + } + } + + // nolint: golint + // 2 because old - new + oldnew := make([]string, 0, 2*len(columnT)) + for t := range columnT { + oldnew = append(oldnew, t, columnT[t]) + } + + r := strings.NewReplacer(oldnew...) + + newQueryOverrides[metric] = r.Replace(newQueryOverrides[metric]) + metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. @@ -571,7 +596,7 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str // TODO: test code for all cu. // TODO: the YAML this supports is "non-standard" - we should move away from it. func addQueries(content []byte, pgVersion semver.Version, server *Server) error { - metricMaps, newQueryOverrides, err := parseUserQueries(content) + metricMaps, newQueryOverrides, err := parseUserQueries(content, pgVersion) if err != nil { return err } diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/cmd/postgres_exporter/postgres_exporter_test.go index 5747f0b07..5645a8ee8 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/cmd/postgres_exporter/postgres_exporter_test.go @@ -410,7 +410,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { func (s *FunctionalSuite) TestParseUserQueries(c *C) { userQueriesData, err := ioutil.ReadFile("./tests/user_queries_ok.yaml") if err == nil { - metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData) + metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData, semver.Version{Major: 13}) c.Assert(err, Equals, nil) c.Assert(metricMaps, NotNil) c.Assert(newQueryOverrides, NotNil) diff --git a/queries.yaml b/queries.yaml index 35b754319..f4f49e609 100644 --- a/queries.yaml +++ b/queries.yaml @@ -159,8 +159,24 @@ pg_database: description: "Disk space used by the database" pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true + breakingChanges: + # should be in asc order by version + - version: '9.4.0' + columns: + total_time_T: total_time + min_time_T: min_time + max_time_T: max_time + mean_time_T: mean_time + stddev_time_T: stddev_time + - version: '13.0.0' + columns: + total_time_T: total_exec_time + min_time_T: min_exec_time + max_time_T: max_exec_time + mean_time_T: mean_exec_time + stddev_time_T: stddev_exec_time metrics: - rolname: usage: "LABEL" From 58d15ed6006cfc0dccac54bdb242e1f705bc4f51 Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 10:29:38 +0300 Subject: [PATCH 2/3] rollback fix Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 34 ---------- cmd/postgres_exporter/postgres_exporter.go | 64 +++++++++++-------- .../postgres_exporter_test.go | 2 +- queries.yaml | 18 +----- 4 files changed, 38 insertions(+), 80 deletions(-) delete mode 100644 cmd/postgres_exporter/pg_breaking_changes.go diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go deleted file mode 100644 index ddd4a76aa..000000000 --- a/cmd/postgres_exporter/pg_breaking_changes.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2021 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "github.com/blang/semver" -) - -type BreakingChanges struct { - Version string `yaml:"version"` - ver semver.Version - Columns map[string]string `yaml:"columns"` -} - -func (bc *BreakingChanges) parseVerTolerant() error { - bcVer, err := semver.ParseTolerant(bc.Version) - if err != nil { - return err - } - - bc.ver = bcVer - return nil -} diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 132e423a7..5bde28c81 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -18,6 +18,7 @@ import ( "database/sql" "errors" "fmt" + "gopkg.in/yaml.v2" "io/ioutil" "math" "net/http" @@ -40,8 +41,6 @@ import ( "github.com/prometheus/common/version" "github.com/prometheus/exporter-toolkit/web" webflag "github.com/prometheus/exporter-toolkit/web/kingpinflag" - "gopkg.in/alecthomas/kingpin.v2" - "gopkg.in/yaml.v2" ) var ( @@ -492,6 +491,39 @@ var queryOverrides = map[string][]OverrideQuery{ `, }, }, + "pg_stat_statements": { + { + semver.MustParseRange(">=13.0.0"), + ` + SELECT t2.rolname, + t3.datname, + queryid, + calls, + total_exec_time / 1000 as total_time_seconds, + min_exec_time / 1000 as min_time_seconds, + max_exec_time / 1000 as max_time_seconds, + mean_exec_time / 1000 as mean_time_seconds, + stddev_exec_time / 1000 as stddev_time_seconds, + rows, + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written, + blk_read_time / 1000 as blk_read_time_seconds, + blk_write_time / 1000 as blk_write_time_seconds + FROM pg_stat_statements t1 + JOIN pg_roles t2 ON (t1.userid = t2.oid) + JOIN pg_database t3 ON (t1.dbid = t3.oid) + WHERE t2.rolname != 'rdsadmin' + `, + }, + }, } // Convert the query override file to the version-specific query override file @@ -518,7 +550,7 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][] return resultMap } -func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]intermediateMetricMap, map[string]string, error) { +func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[string]string, error) { var userQueries UserQueries err := yaml.Unmarshal(content, &userQueries) @@ -533,30 +565,6 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - columnT := make(map[string]string) - for i := range specs.BreakingChanges { - if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { - return nil, nil, err - } - - if pgVersion.GE(specs.BreakingChanges[i].ver) { - for t := range specs.BreakingChanges[i].Columns { - columnT[t] = specs.BreakingChanges[i].Columns[t] - } - } - } - - // nolint: golint - // 2 because old - new - oldnew := make([]string, 0, 2*len(columnT)) - for t := range columnT { - oldnew = append(oldnew, t, columnT[t]) - } - - r := strings.NewReplacer(oldnew...) - - newQueryOverrides[metric] = r.Replace(newQueryOverrides[metric]) - metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. @@ -596,7 +604,7 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte // TODO: test code for all cu. // TODO: the YAML this supports is "non-standard" - we should move away from it. func addQueries(content []byte, pgVersion semver.Version, server *Server) error { - metricMaps, newQueryOverrides, err := parseUserQueries(content, pgVersion) + metricMaps, newQueryOverrides, err := parseUserQueries(content) if err != nil { return err } diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/cmd/postgres_exporter/postgres_exporter_test.go index 5645a8ee8..5747f0b07 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/cmd/postgres_exporter/postgres_exporter_test.go @@ -410,7 +410,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { func (s *FunctionalSuite) TestParseUserQueries(c *C) { userQueriesData, err := ioutil.ReadFile("./tests/user_queries_ok.yaml") if err == nil { - metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData, semver.Version{Major: 13}) + metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData) c.Assert(err, Equals, nil) c.Assert(metricMaps, NotNil) c.Assert(newQueryOverrides, NotNil) diff --git a/queries.yaml b/queries.yaml index f4f49e609..35b754319 100644 --- a/queries.yaml +++ b/queries.yaml @@ -159,24 +159,8 @@ pg_database: description: "Disk space used by the database" pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true - breakingChanges: - # should be in asc order by version - - version: '9.4.0' - columns: - total_time_T: total_time - min_time_T: min_time - max_time_T: max_time - mean_time_T: mean_time - stddev_time_T: stddev_time - - version: '13.0.0' - columns: - total_time_T: total_exec_time - min_time_T: min_exec_time - max_time_T: max_exec_time - mean_time_T: mean_exec_time - stddev_time_T: stddev_exec_time metrics: - rolname: usage: "LABEL" From 36ec34339c0f8f15ac8e9634e82b1dda5e0b9d9b Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 10:32:59 +0300 Subject: [PATCH 3/3] fix unused Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 5bde28c81..cecf8f601 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -18,6 +18,7 @@ import ( "database/sql" "errors" "fmt" + "gopkg.in/alecthomas/kingpin.v2" "gopkg.in/yaml.v2" "io/ioutil" "math" @@ -118,12 +119,11 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - BreakingChanges []BreakingChanges `yaml:"breakingChanges"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint