diff --git a/cmd/pipecd/BUILD.bazel b/cmd/pipecd/BUILD.bazel index 81d33ac1bf..2afd08d76f 100644 --- a/cmd/pipecd/BUILD.bazel +++ b/cmd/pipecd/BUILD.bazel @@ -26,6 +26,7 @@ go_library( "//pkg/app/ops/insightcollector:go_default_library", "//pkg/app/ops/mysqlensurer:go_default_library", "//pkg/app/ops/orphancommandcleaner:go_default_library", + "//pkg/app/ops/pipedstatsbuilder:go_default_library", "//pkg/cache/cachemetrics:go_default_library", "//pkg/cache/rediscache:go_default_library", "//pkg/cli:go_default_library", diff --git a/cmd/pipecd/ops.go b/cmd/pipecd/ops.go index 3064062143..02aac49232 100644 --- a/cmd/pipecd/ops.go +++ b/cmd/pipecd/ops.go @@ -29,11 +29,14 @@ import ( "github.com/pipe-cd/pipe/pkg/app/ops/insightcollector" "github.com/pipe-cd/pipe/pkg/app/ops/mysqlensurer" "github.com/pipe-cd/pipe/pkg/app/ops/orphancommandcleaner" + "github.com/pipe-cd/pipe/pkg/app/ops/pipedstatsbuilder" + "github.com/pipe-cd/pipe/pkg/cache/rediscache" "github.com/pipe-cd/pipe/pkg/cli" "github.com/pipe-cd/pipe/pkg/config" "github.com/pipe-cd/pipe/pkg/datastore" "github.com/pipe-cd/pipe/pkg/insight/insightstore" "github.com/pipe-cd/pipe/pkg/model" + "github.com/pipe-cd/pipe/pkg/redis" "github.com/pipe-cd/pipe/pkg/version" ) @@ -44,13 +47,15 @@ type ops struct { enableInsightCollector bool configFile string gcloudPath string + cacheAddress string } func NewOpsCommand() *cobra.Command { s := &ops{ - httpPort: 9082, - adminPort: 9085, - gracePeriod: 15 * time.Second, + httpPort: 9082, + adminPort: 9085, + cacheAddress: "cache:6379", + gracePeriod: 15 * time.Second, } cmd := &cobra.Command{ Use: "ops", @@ -62,6 +67,7 @@ func NewOpsCommand() *cobra.Command { cmd.Flags().DurationVar(&s.gracePeriod, "grace-period", s.gracePeriod, "How long to wait for graceful shutdown.") cmd.Flags().StringVar(&s.configFile, "config-file", s.configFile, "The path to the configuration file.") cmd.Flags().StringVar(&s.gcloudPath, "gcloud-path", s.gcloudPath, "The path to the gcloud command executable.") + cmd.Flags().StringVar(&s.cacheAddress, "cache-address", s.cacheAddress, "The address to cache service.") return cmd } @@ -144,6 +150,10 @@ func (s *ops) run(ctx context.Context, t cli.Telemetry) error { }) } + rd := redis.NewRedis(s.cacheAddress, "") + statCache := rediscache.NewTTLHashCache(rd, cfg.Cache.TTLDuration(), defaultPipedStatHashKey) + psb := pipedstatsbuilder.NewPipedStatsBuilder(statCache, t.Logger) + // Start running admin server. { var ( @@ -157,7 +167,7 @@ func (s *ops) run(ctx context.Context, t cli.Telemetry) error { admin.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }) - admin.Handle("/metrics", t.PrometheusMetricsHandler()) + admin.Handle("/metrics", t.CustomMetricsHandlerFor(psb)) group.Go(func() error { return admin.Run(ctx) diff --git a/manifests/pipecd/templates/deployment.yaml b/manifests/pipecd/templates/deployment.yaml index 48ead1dac8..fa4ce937b3 100644 --- a/manifests/pipecd/templates/deployment.yaml +++ b/manifests/pipecd/templates/deployment.yaml @@ -215,6 +215,7 @@ spec: imagePullPolicy: IfNotPresent args: - ops + - --cache-address={{ .Values.ops.args.cacheAddress | default (printf "%s-cache:6379" (include "pipecd.fullname" .)) }} - --config-file=/etc/pipecd-config/{{ .Values.config.fileName }} - --log-encoding={{ .Values.ops.args.logEncoding }} ports: diff --git a/manifests/pipecd/values.yaml b/manifests/pipecd/values.yaml index 0759fb91f2..4096a73307 100644 --- a/manifests/pipecd/values.yaml +++ b/manifests/pipecd/values.yaml @@ -40,6 +40,7 @@ ops: image: repository: gcr.io/pipecd/pipecd args: + cacheAddress: "" logEncoding: humanize resources: {} diff --git a/pkg/app/ops/pipedstatsbuilder/BUILD.bazel b/pkg/app/ops/pipedstatsbuilder/BUILD.bazel new file mode 100644 index 0000000000..946cb902d5 --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/BUILD.bazel @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["builder.go"], + importpath = "github.com/pipe-cd/pipe/pkg/app/ops/pipedstatsbuilder", + visibility = ["//visibility:public"], + deps = [ + "//pkg/cache:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) + +go_test( + name = "go_default_test", + size = "small", + srcs = ["builder_test.go"], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = [ + "//pkg/cache:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) diff --git a/pkg/app/ops/pipedstatsbuilder/builder.go b/pkg/app/ops/pipedstatsbuilder/builder.go new file mode 100644 index 0000000000..aae5fdd711 --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/builder.go @@ -0,0 +1,56 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipedstatsbuilder + +import ( + "bytes" + "errors" + "io" + + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/cache" +) + +type PipedStatsBuilder struct { + backend cache.Cache + logger *zap.Logger +} + +func NewPipedStatsBuilder(c cache.Cache, logger *zap.Logger) *PipedStatsBuilder { + return &PipedStatsBuilder{ + backend: c, + logger: logger.Named("piped-metrics-builder"), + } +} + +func (b *PipedStatsBuilder) Build() (io.Reader, error) { + res, err := b.backend.GetAll() + if err != nil { + b.logger.Error("failed to fetch piped stats from cache", zap.Error(err)) + return nil, err + } + data := make([][]byte, 0, len(res)) + for _, v := range res { + value, okValue := v.([]byte) + if !okValue { + err = errors.New("error value not a bulk of string value") + b.logger.Error("failed to marshal piped stat data", zap.Error(err)) + return nil, err + } + data = append(data, value) + } + return bytes.NewReader(bytes.Join(data, []byte("\n"))), nil +} diff --git a/pkg/app/ops/pipedstatsbuilder/builder_test.go b/pkg/app/ops/pipedstatsbuilder/builder_test.go new file mode 100644 index 0000000000..e391738c84 --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/builder_test.go @@ -0,0 +1,65 @@ +// Copyright 2021 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipedstatsbuilder + +import ( + "io" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/cache" +) + +type mockBuilderBackend struct { + cache.Cache + srcs []string +} + +func newMockBuilderBackend() *mockBuilderBackend { + return &mockBuilderBackend{ + srcs: []string{ + "./testdata/piped_stat_1", + "./testdata/piped_stat_2", + }, + } +} + +func (m *mockBuilderBackend) GetAll() (map[string]interface{}, error) { + out := make(map[string]interface{}, len(m.srcs)) + for _, file := range m.srcs { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + out[file] = data + } + return out, nil +} + +func TestBuildPipedStat(t *testing.T) { + builder := NewPipedStatsBuilder(newMockBuilderBackend(), zap.NewNop()) + rc, err := builder.Build() + require.NoError(t, err) + require.NotNil(t, rc) + buf := new(strings.Builder) + io.Copy(buf, rc) + data, _ := os.ReadFile("./testdata/expected") + assert.Equal(t, string(data), buf.String()) +} diff --git a/pkg/app/ops/pipedstatsbuilder/testdata/expected b/pkg/app/ops/pipedstatsbuilder/testdata/expected new file mode 100644 index 0000000000..75404b2e46 --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/testdata/expected @@ -0,0 +1,181 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0"} 8.3886e-05 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.25"} 0.000392513 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.5"} 0.000472378 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.75"} 0.000596689 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="1"} 0.001621264 +go_gc_duration_seconds_sum{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 0.036167819 +go_gc_duration_seconds_count{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 71 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 36 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",version="go1.16.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.65208088e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.480348e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 686049 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 6.293948048802884e-06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.700728e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.5296e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.0272768e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 31290 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.3633024e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 6.5568768e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.625729975673197e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 717339 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 32768 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 282880 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 294912 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.2980992e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 3.584244e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.8201864e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 23 + +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0"} 8.3886e-05 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.25"} 0.000392513 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.5"} 0.000472378 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.75"} 0.000596689 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="1"} 0.001621264 +go_gc_duration_seconds_sum{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 0.036167819 +go_gc_duration_seconds_count{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 71 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 36 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",version="go1.16.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.65208088e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.480348e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 686049 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 6.293948048802884e-06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.700728e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.5296e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.0272768e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 31290 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.3633024e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 6.5568768e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.625729975673197e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 717339 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 32768 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 282880 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 294912 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.2980992e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 3.584244e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.8201864e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 23 diff --git a/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_1 b/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_1 new file mode 100644 index 0000000000..12bc3d4d5a --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_1 @@ -0,0 +1,90 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0"} 8.3886e-05 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.25"} 0.000392513 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.5"} 0.000472378 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="0.75"} 0.000596689 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",quantile="1"} 0.001621264 +go_gc_duration_seconds_sum{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 0.036167819 +go_gc_duration_seconds_count{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 71 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 36 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified",version="go1.16.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.65208088e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.480348e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 686049 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 6.293948048802884e-06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.700728e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.5296e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.0272768e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 31290 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 5.3633024e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 6.5568768e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.625729975673197e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 717339 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 32768 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 282880 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 294912 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.2980992e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 3.584244e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 7.8201864e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb2a",piped_version="unspecified"} 23 diff --git a/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_2 b/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_2 new file mode 100644 index 0000000000..044367cc2d --- /dev/null +++ b/pkg/app/ops/pipedstatsbuilder/testdata/piped_stat_2 @@ -0,0 +1,90 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0"} 8.3886e-05 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.25"} 0.000392513 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.5"} 0.000472378 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="0.75"} 0.000596689 +go_gc_duration_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",quantile="1"} 0.001621264 +go_gc_duration_seconds_sum{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 0.036167819 +go_gc_duration_seconds_count{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 71 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 36 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified",version="go1.16.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.65208088e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.480348e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 686049 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 6.293948048802884e-06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.700728e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.839528e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.5296e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.0272768e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 31290 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 5.3633024e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 6.5568768e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.625729975673197e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 717339 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 32768 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 282880 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 294912 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.2980992e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 3.584244e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 1.540096e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 7.8201864e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads{piped="763a5051-f1a2-4160-bfdf-6b0070d3cb1b",piped_version="unspecified"} 23 diff --git a/pkg/cli/cmd.go b/pkg/cli/cmd.go index 50345e3cdb..6e136efab7 100644 --- a/pkg/cli/cmd.go +++ b/pkg/cli/cmd.go @@ -16,6 +16,7 @@ package cli import ( "context" + "io" "net/http" "os" "os/signal" @@ -137,6 +138,30 @@ func (t Telemetry) PrometheusMetricsHandlerFor(r *prometheus.Registry) http.Hand return empty } +type MetricsBuilder interface { + Build() (io.Reader, error) +} + +func (t Telemetry) CustomMetricsHandlerFor(mb MetricsBuilder) http.Handler { + if t.Flags.Metrics { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rc, err := mb.Build() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + _, err = io.Copy(w, rc) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + } + var empty http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("")) + } + return empty +} + func extractServiceName(cmd *cobra.Command) string { return strings.Replace(cmd.CommandPath(), " ", ".", -1) }