From 5c3d734c8c7ffc14a34333e02791ea904c97070b Mon Sep 17 00:00:00 2001 From: sgayangi Date: Mon, 19 Feb 2024 16:04:34 +0530 Subject: [PATCH 01/10] Add prometheus configs to adapter enforcer router --- adapter/config/default_config.go | 6 + adapter/config/types.go | 8 +- adapter/go.mod | 6 + adapter/go.sum | 20 ++ adapter/internal/adapter/adapter.go | 9 + adapter/pkg/metrics/metrics.go | 215 ++++++++++++++++++ adapter/pkg/metrics/types.go | 24 ++ .../org.wso2.apk.enforcer/build.gradle | 1 + .../org/wso2/apk/enforcer/jmx/JMXUtils.java | 4 +- .../metrics/jmx/api/ExtAuthMetricsMXBean.java | 36 +-- .../metrics/jmx/impl/ExtAuthMetrics.java | 67 +++--- .../metrics/prometheus-jmx-config.yml | 44 ++++ .../adapter/adapter-deployment.yaml | 4 + .../adapter/adapter-service.yaml | 5 + .../gateway-runtime/enforcer-service.yaml | 5 + .../gateway-runtime-deployment.yaml | 22 +- .../prometheus-jmx-configmap.yaml | 31 +++ .../gateway-runtime/router-service.yaml | 31 +++ helm-charts/values.yaml | 3 + libs.versions.toml | 2 + 20 files changed, 495 insertions(+), 48 deletions(-) create mode 100644 adapter/pkg/metrics/metrics.go create mode 100644 adapter/pkg/metrics/types.go create mode 100644 gateway/enforcer/resources/metrics/prometheus-jmx-config.yml create mode 100644 helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml create mode 100644 helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 86630ad57..1fe3847d7 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -32,6 +32,12 @@ var defaultConfig = &Config{ Namespaces: nil, }, Environment: "Default", + Metrics: metrics{ + Enabled: true, + Type: "prometheus", + Port: 18006, + CollectionInterval: 5, + }, }, Envoy: envoy{ ListenerCodecType: "AUTO", diff --git a/adapter/config/types.go b/adapter/config/types.go index f51f2577f..8e8954f38 100644 --- a/adapter/config/types.go +++ b/adapter/config/types.go @@ -93,6 +93,8 @@ type adapter struct { Operator operator // Environment of the Adapter Environment string + // Metric represents configurations to expose/export go metrics + Metrics metrics } // Envoy Listener Component related configurations. @@ -299,8 +301,10 @@ type tracing struct { } type metrics struct { - Enabled bool - Type string + Enabled bool + Type string + Port int32 + CollectionInterval int32 } type analyticsAdapter struct { diff --git a/adapter/go.mod b/adapter/go.mod index b276c876e..d90a94e1d 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -10,6 +10,9 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/pelletier/go-toml v1.9.5 + github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/procfs v0.12.0 + github.com/shirou/gopsutil/v3 v3.24.1 github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb @@ -47,6 +50,7 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -59,6 +63,8 @@ require ( github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/vektah/gqlparser v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index 26378461c..c943d5bd9 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -62,6 +62,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -88,6 +89,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -112,6 +115,11 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -121,6 +129,12 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= +github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -136,6 +150,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU= github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -182,9 +200,11 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 08ba6d809..83575ee23 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -20,6 +20,7 @@ package adapter import ( "crypto/tls" + "strings" "time" discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" @@ -33,6 +34,7 @@ import ( wso2_server "github.com/wso2/apk/adapter/pkg/discovery/protocol/server/v3" "github.com/wso2/apk/adapter/pkg/health" healthservice "github.com/wso2/apk/adapter/pkg/health/api/wso2/health/service" + metrics "github.com/wso2/apk/adapter/pkg/metrics" "github.com/wso2/apk/adapter/pkg/utils/tlsutils" "context" @@ -154,6 +156,13 @@ func Run(conf *config.Config) { } logger.LoggerAPK.Info("Starting adapter ....") + + // Start the metrics server + if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { + logger.LoggerAPK.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) + } + cache := xds.GetXdsCache() enforcerCache := xds.GetEnforcerCache() enforcerAPICache := xds.GetEnforcerAPICache() diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go new file mode 100644 index 000000000..62a82e254 --- /dev/null +++ b/adapter/pkg/metrics/metrics.go @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "net/http" + "os" + "runtime" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/procfs" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + logger "github.com/wso2/apk/adapter/internal/loggers" + "github.com/wso2/apk/adapter/pkg/logging" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() + + hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "host_info", + Help: "Host Info", + }, []string{"os"}) + + availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_available_cpu_total", + Help: "Number of available CPUs.", + }) + + freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_free_physical_memory_bytes", + Help: "Amount of free physical memory.", + }) + + totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_total_virtual_memory_bytes", + Help: "Amount of total virtual memory.", + }) + usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_used_virtual_memory_bytes", + Help: "Amount of used virtual memory.", + }) + + systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_system_cpu_load_percentage", + Help: "System-wide CPU usage as a percentage.", + }) + + loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "os_system_load_average", + Help: "Current load of CPU in the host system for the last {x} minutes", + }, []string{"duration"}) + + processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_start_time_seconds", + Help: "Start time of the process since unix epoch in seconds.", + }) + + processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_open_fds", + Help: "Number of open file descriptors.", + }) +) + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) + + // Register other metrics + prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, + systemCPULoad, loadAvg, processStartTime, processOpenFDs) +} + +// recordMetrics record custom golang metrics +var recordMetrics = func(collectionInterval int32) { + for { + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfo.WithLabelValues(host.OS).Set(1) + availableCPUs.Set(float64(runtime.NumCPU())) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory.Set(float64(v.Free)) + usedVirtualMemory.Set(float64(v.Used)) + totalVirtualMemory.Set(float64(v.Total)) + + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad.Set(averagePercentage) + + pid := os.Getpid() + p, err := procfs.NewProc(pid) + if handleError(err, "Failed to get current process") { + return + } + stat, err := p.Stat() + if handleError(err, "Failed to get process stats") { + return + } + t, err := stat.StartTime() + if handleError(err, "Failed to read process start time") { + return + } + processStartTime.Set(t) + fds, err := p.FileDescriptorsLen() + if handleError(err, "Failed to read file descriptor count") { + return + } + processOpenFDs.Set(float64(fds)) + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + loadAvg.WithLabelValues("1m").Set(avg.Load1) + loadAvg.WithLabelValues("5m").Set(avg.Load5) + loadAvg.WithLabelValues("15m").Set(avg.Load15) + + // Sleep before the next measurement + time.Sleep(time.Duration(collectionInterval) * time.Second) + } + +} + +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) + return true + } + return false +} + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32, collectionInterval int32) { + done := make(chan struct{}) // Channel to indicate recordMetrics routine exit + + // Start the Prometheus metrics server + go func() { + http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } + }() + + for { + // Start the recordMetrics goroutine + go func() { + defer func() { + if r := recover(); r != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), + Severity: logging.MAJOR, + ErrorCode: 1111, + }) + } + }() + recordMetrics(collectionInterval) + done <- struct{}{} // Signal that the goroutine has completed + }() + + // Wait for the previous recordMetrics goroutine to complete + <-done + + // Log and restart the goroutine + logger.LoggerAPK.Info("Restarting recordMetrics goroutine...") + time.Sleep(3 * time.Second) + } +} \ No newline at end of file diff --git a/adapter/pkg/metrics/types.go b/adapter/pkg/metrics/types.go new file mode 100644 index 000000000..8e9fae144 --- /dev/null +++ b/adapter/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package metrics + + const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" + ) \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle index 8d1b50f11..5a7bc193a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle +++ b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle @@ -71,6 +71,7 @@ dependencies { implementation libs.opentelemetry.extension.trace.propagators implementation libs.opentelemetry.sdk implementation libs.opentelemetry.semconv + implementation libs.prometheus implementation libs.snakeyaml implementation libs.sun.saaj.impl implementation libs.swagger.core.v3 diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java index 2c2028c8a..f460437ae 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java @@ -21,7 +21,7 @@ */ public class JMXUtils { - private static final String CHOREO_CONNECT_JMX_METRICS_ENABLE = "choreo.connect.jmx.metrics.enabled"; + private static final String APK_JMX_METRICS_ENABLE = "apk.jmx.metrics.enabled"; /** * Returns true if jmx metrics enabled as a system property, otherwise false. @@ -29,6 +29,6 @@ public class JMXUtils { * @return boolean */ public static boolean isJMXMetricsEnabled() { - return Boolean.getBoolean(CHOREO_CONNECT_JMX_METRICS_ENABLE); + return Boolean.getBoolean(APK_JMX_METRICS_ENABLE); } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java index 5bd3fc919..105eccf13 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -23,31 +23,32 @@ public interface ExtAuthMetricsMXBean { /** * Getter for total request count. - * + * * @return long */ public long getTotalRequestCount(); /** * Getter for average response time in milli seconds. - * - * @return long + * + * @return double */ - public long getAverageResponseTimeMillis(); + public double getAverageResponseTimeMillis(); /** * Getter for maximum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMaxResponseTimeMillis(); + public double getMaxResponseTimeMillis(); + /** * Getter for mimnimum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMinResponseTimeMillis(); + public double getMinResponseTimeMillis(); /** * Resets all the metrics to thier initial values. @@ -55,8 +56,17 @@ public interface ExtAuthMetricsMXBean { public void resetExtAuthMetrics(); /** - * Resets all the metrics to thier initial values. + * Get request count in last five minutes window + * + * @return long + */ + public long getRequestCountInLastFiveMinuteWindow(); + + /** + * Get the start time of request count window + * + * @return */ - public long getRequestCountInLastFiveMinutes(); + public long getRequestCountWindowStartTimeMillis(); -} +} \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index 9f558ba1f..e01ac205c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -30,33 +30,35 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; private static ExtAuthMetrics extAuthMetricsMBean = null; - private long requestCountInLastFiveMinutes = 0; + private long requestCountInLastFiveMinuteWindow = 0; + private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); private long totalRequestCount = 0; - private long averageResponseTimeMillis = 0; - private long maxResponseTimeMillis = Long.MIN_VALUE; - private long minResponseTimeMillis = Long.MAX_VALUE; + private double averageResponseTimeMillis = 0; + private double maxResponseTimeMillis = Double.MIN_VALUE; + private double minResponseTimeMillis = Double.MAX_VALUE; private ExtAuthMetrics() { MBeanRegistrator.registerMBean(this); } - /** - * Getter for the Singleton ExtAuthMetrics instance. - * - * @return ExtAuthMetrics - */ - public static ExtAuthMetrics getInstance() { - if (extAuthMetricsMBean == null) { - synchronized (ExtAuthMetrics.class) { - if (extAuthMetricsMBean == null) { - Timer timer = new Timer(); - extAuthMetricsMBean = new ExtAuthMetrics(); - timer.schedule(extAuthMetricsMBean, 0, REQUEST_COUNT_INTERVAL_MILLIS); - } +/** + * Getter for the Singleton ExtAuthMetrics instance. + * + * @return ExtAuthMetrics + */ +public static ExtAuthMetrics getInstance() { + if (extAuthMetricsMBean == null) { + synchronized (ExtAuthMetrics.class) { + if (extAuthMetricsMBean == null) { + Timer timer = new Timer(); + extAuthMetricsMBean = new ExtAuthMetrics(); + extAuthMetricsMBean.requestCountWindowStartTimeMillis = System.currentTimeMillis(); + timer.schedule(extAuthMetricsMBean, REQUEST_COUNT_INTERVAL_MILLIS, REQUEST_COUNT_INTERVAL_MILLIS); } } - return extAuthMetricsMBean; } + return extAuthMetricsMBean; +} @Override public long getTotalRequestCount() { @@ -64,24 +66,25 @@ public long getTotalRequestCount() { }; @Override - public long getAverageResponseTimeMillis() { + public double getAverageResponseTimeMillis() { return averageResponseTimeMillis; }; @Override - public long getMaxResponseTimeMillis() { + public double getMaxResponseTimeMillis() { return maxResponseTimeMillis; }; @Override - public long getMinResponseTimeMillis() { + public double getMinResponseTimeMillis() { return minResponseTimeMillis; }; public synchronized void recordMetric(long responseTimeMillis) { - this.requestCountInLastFiveMinutes += 1; + this.requestCountInLastFiveMinuteWindow += 1; this.totalRequestCount += 1; - this.averageResponseTimeMillis = (this.averageResponseTimeMillis + responseTimeMillis) / totalRequestCount; + this.averageResponseTimeMillis = this.averageResponseTimeMillis + + (responseTimeMillis - this.averageResponseTimeMillis) / totalRequestCount; this.minResponseTimeMillis = Math.min(this.minResponseTimeMillis, responseTimeMillis); this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); } @@ -90,17 +93,23 @@ public synchronized void recordMetric(long responseTimeMillis) { public synchronized void resetExtAuthMetrics() { this.totalRequestCount = 0; this.averageResponseTimeMillis = 0; - this.maxResponseTimeMillis = Long.MIN_VALUE; - this.minResponseTimeMillis = Long.MAX_VALUE; + this.maxResponseTimeMillis = Double.MIN_VALUE; + this.minResponseTimeMillis = Double.MAX_VALUE; } @Override public synchronized void run() { - requestCountInLastFiveMinutes = 0; + requestCountWindowStartTimeMillis = System.currentTimeMillis(); + requestCountInLastFiveMinuteWindow = 0; } @Override - public long getRequestCountInLastFiveMinutes() { - return requestCountInLastFiveMinutes; + public long getRequestCountInLastFiveMinuteWindow() { + return requestCountInLastFiveMinuteWindow; } -} + + @Override + public long getRequestCountWindowStartTimeMillis() { + return requestCountWindowStartTimeMillis; + } +} \ No newline at end of file diff --git a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml new file mode 100644 index 000000000..ef6c6c86e --- /dev/null +++ b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +# Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + # WSO2 APK related metrics + - pattern: 'org.wso2.apk.enforcer<>total_request_count: (.*)' + name: org_wso2_apk_enforcer_request_count_total + help: "WSO2 APK enforcer total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + name: org_wso2_apk_enforcer_$1 + help: "WSO2 APK enforcer $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' + name: org_wso2_apk_enforcer_thread_pool_$1 + help: "WSO2 APK enforcer thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE \ No newline at end of file diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml index 1f28bf973..7bf0a25fc 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml @@ -46,6 +46,10 @@ spec: protocol: "TCP" - containerPort: 18001 protocol: TCP + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + - containerPort: 18006 + protocol: "TCP" + {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.adapter.deployment.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.adapter.deployment.env | indent 10 }} - name: OPERATOR_POD_NAMESPACE diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml index 8c78ff45c..c0c8e0c7c 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml @@ -41,4 +41,9 @@ spec: - name: profile-port protocol: TCP port: 6060 + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + - name: prometheus-adapter-port + port: 18006 + protocol: "TCP" + {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml index e88b6d861..a8355ce99 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml @@ -28,4 +28,9 @@ spec: - name: servlet-https protocol: TCP port: 9092 + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-enforcer-port + protocol: TCP + port: 18007 + {{- end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index a9918a9a5..2a0a055d5 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -52,6 +52,10 @@ spec: protocol: "TCP" - containerPort: 18002 protocol: "TCP" + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + - containerPort: 18007 + protocol: "TCP" + {{- end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.env | indent 10 }} - name: ADAPTER_HOST_NAME @@ -87,7 +91,11 @@ spec: - name: enforcer_admin_pwd value: admin - name: JAVA_OPTS - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config.yml + {{- else }} + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + {{- end }} {{- if .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.redis }} - name: REDIS_USERNAME value: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.redis.username | default "default" }} @@ -153,6 +161,11 @@ spec: {{- else }} subPath: tls.key {{- end }} + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config.yml + subPath: prometheus-jmx-config.yml + {{- end }} - name: enforcer-keystore-secret-volume mountPath: /home/wso2/security/keystore/enforcer.crt {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} @@ -382,9 +395,14 @@ spec: # secretName: {{ template "apk-helm.resource.prefix" . }}-common-controller-server-cert # {{- end }} # defaultMode: 420 + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config + {{- end }} - name: enforcer-keystore-secret-volume secret: - {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} + {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} secretName: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls.certificatesSecret | default (printf "%s-enforcer-server-cert" (include "apk-helm.resource.prefix" .)) }} {{- else }} secretName: {{ template "apk-helm.resource.prefix" . }}-enforcer-server-cert diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..71065b996 --- /dev/null +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config +data: + prometheus-jmx-config.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # WSO2 APK related metrics + - pattern: 'org.wso2.apk.enforcer<>total_request_count: (.*)' + name: org_wso2_apk_enforcer_request_count_total + help: "WSO2 APK enforcer total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + name: org_wso2_apk_enforcer_$1 + help: "WSO2 APK enforcer $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' + name: org_wso2_apk_enforcer_thread_pool_$1 + help: "WSO2 APK enforcer thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE \ No newline at end of file diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml new file mode 100644 index 000000000..8450a504d --- /dev/null +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "apk-helm.resource.prefix" . }}-router-service + namespace : {{ .Release.Namespace }} +spec: + # label keys and values that must match in order to receive traffic for this service + selector: +{{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "gateway" ) | indent 4}} + ports: + - name: prometheus-router-port + protocol: TCP + port: 9000 +{{- end -}} diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index dadb28be3..00976c673 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -249,6 +249,9 @@ wso2: # JWKSClient: # skipSSLVerification: false # hostnameVerifier: "AllowAll" + + metrics: + enabled: true idp: enabled: true listener: diff --git a/libs.versions.toml b/libs.versions.toml index 98a8315ee..100dd115a 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -80,6 +80,7 @@ opentelemetry-extension-trace-propagators = {module = "io.opentelemetry:opentele opentelemetry-sdk = {module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry"} opentelemetry-semconv = {module = "io.opentelemetry:opentelemetry-semconv", version.ref = "opentelemetry-semconv"} postgresql = {module = "org.postgresql:postgresql", version.ref = "postgresql"} +prometheus = {module = "io.prometheus.jmx:jmx_prometheus_javaagent", version.ref = "prometheus"} snakeyaml = {module = "org.yaml:snakeyaml", version.ref = "snakeyaml"} sun-saaj-impl = {module = "com.sun.xml.messaging.saaj:saaj-impl", version.ref = "sun"} swagger-annotations = {module = "io.swagger:swagger-annotations", version.ref = "io-swagger"} @@ -165,6 +166,7 @@ opentelemetry-jaeger-thrift = "1.24.0" opentelemetry-semconv = "1.24.0-alpha" org-json = "20231013" postgresql = "42.5.0" +prometheus = "0.20.0" snakeyaml = "2.0" sun = "1.5.3" swagger-codegen = "2.4.11" From 67660fa685d0f114ae2aba6a4d85075d19954633 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Sun, 25 Feb 2024 09:20:51 +0530 Subject: [PATCH 02/10] Add Prometheus support to common controller --- adapter/go.mod | 6 +- adapter/go.sum | 10 +- .../commoncontroller/common_controller.go | 8 + common-controller/go.mod | 12 +- common-controller/go.sum | 28 ++- .../internal/config/default_config.go | 9 +- common-controller/internal/config/types.go | 8 + common-controller/pkg/metrics/metrics.go | 215 ++++++++++++++++++ common-controller/pkg/metrics/types.go | 24 ++ .../discovery/JWTIssuerDiscoveryClient.java | 6 + .../apk/enforcer/grpc/ExtAuthService.java | 6 +- .../metrics/jmx/api/ExtAuthMetricsMXBean.java | 17 +- .../metrics/jmx/impl/ExtAuthMetrics.java | 21 +- .../SubscriptionDataStoreUtil.java | 5 + .../metrics/prometheus-jmx-config.yml | 6 +- .../config-deployer/config-ds-configmap.yaml | 12 + .../common-controller-deployment.yaml | 4 + .../common-controller-service.yaml | 5 + .../gateway-runtime-deployment.yaml | 2 +- .../prometheus-jmx-configmap.yaml | 2 +- .../artifact-deployer-api_service.bal | 10 +- 21 files changed, 391 insertions(+), 25 deletions(-) create mode 100644 common-controller/pkg/metrics/metrics.go create mode 100644 common-controller/pkg/metrics/types.go diff --git a/adapter/go.mod b/adapter/go.mod index d90a94e1d..03faae8b7 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -37,6 +37,7 @@ require ( github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -58,14 +59,15 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/vektah/gqlparser v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.21.0 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index c943d5bd9..0af7a43be 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -41,6 +41,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -113,13 +115,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -160,6 +159,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -211,6 +212,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index ea071545c..676c87bff 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -25,6 +25,7 @@ import ( "net" "os" "os/signal" + "strings" "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -40,6 +41,7 @@ import ( "github.com/wso2/apk/common-controller/internal/server" utils "github.com/wso2/apk/common-controller/internal/utils" xds "github.com/wso2/apk/common-controller/internal/xds" + "github.com/wso2/apk/common-controller/pkg/metrics" apkmgt "github.com/wso2/apk/common-go-libs/pkg/discovery/api/wso2/discovery/service/apkmgt" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -206,6 +208,12 @@ func InitCommonControllerServer(conf *config.Config) { loggers.LoggerAPKOperator.Info("Starting common controller ....") + // Start the metrics server + if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { + loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port, conf.CommonController.Metrics.CollectionInterval) + } + rateLimiterCache := xds.GetRateLimiterCache() rlsSrv := xdsv3.NewServer(ctx, rateLimiterCache, &xds.Callbacks{}) diff --git a/common-controller/go.mod b/common-controller/go.mod index 80b975191..9013a7ac6 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -5,6 +5,8 @@ go 1.20 require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 + github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/procfs v0.12.0 github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -18,6 +20,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 + github.com/shirou/gopsutil/v3 v3.24.1 github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 google.golang.org/grpc v1.62.0 @@ -34,6 +37,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect @@ -41,13 +45,19 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vektah/gqlparser v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect @@ -84,10 +94,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 go.uber.org/multierr v1.11.0 // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index da9f79556..b678ced88 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -66,6 +66,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -103,6 +105,7 @@ github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvR github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -138,6 +141,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= @@ -165,8 +170,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= @@ -179,6 +186,12 @@ github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= +github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -200,6 +213,10 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= @@ -210,6 +227,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -270,9 +289,11 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -281,6 +302,9 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 081122c36..2391f9524 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -41,6 +41,13 @@ var defaultConfig = &Config{ EventPort: 18000, RestPort: 18001, RetryInterval: 5, - Persistence: persistence{Type: "K8s"}}, + Persistence: persistence{Type: "K8s"}, + }, + Metrics: metrics{ + Enabled: true, + Type: "prometheus", + Port: 18009, + CollectionInterval: 5, + }, }, } diff --git a/common-controller/internal/config/types.go b/common-controller/internal/config/types.go index 372dc195e..35b57dd32 100644 --- a/common-controller/internal/config/types.go +++ b/common-controller/internal/config/types.go @@ -48,6 +48,7 @@ type commoncontroller struct { WebServer webServer InternalAPIServer internalAPIServer ControlPlane controlplane + Metrics metrics } type controlplane struct { Enabled bool @@ -102,3 +103,10 @@ type sts struct { type webServer struct { Port int64 } + +type metrics struct { + Enabled bool + Type string + Port int32 + CollectionInterval int32 +} diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go new file mode 100644 index 000000000..ef7824ebb --- /dev/null +++ b/common-controller/pkg/metrics/metrics.go @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing common controller metrics to prometheus +package metrics + +import ( + "fmt" + "net/http" + "os" + "runtime" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/procfs" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/wso2/apk/adapter/pkg/logging" + logger "github.com/wso2/apk/common-controller/internal/loggers" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() + + hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "host_info", + Help: "Host Info", + }, []string{"os"}) + + availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_available_cpu_total", + Help: "Number of available CPUs.", + }) + + freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_free_physical_memory_bytes", + Help: "Amount of free physical memory.", + }) + + totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_total_virtual_memory_bytes", + Help: "Amount of total virtual memory.", + }) + usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_used_virtual_memory_bytes", + Help: "Amount of used virtual memory.", + }) + + systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "os_system_cpu_load_percentage", + Help: "System-wide CPU usage as a percentage.", + }) + + loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "os_system_load_average", + Help: "Current load of CPU in the host system for the last {x} minutes", + }, []string{"duration"}) + + processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_start_time_seconds", + Help: "Start time of the process since unix epoch in seconds.", + }) + + processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_open_fds", + Help: "Number of open file descriptors.", + }) +) + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) + + // Register other metrics + prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, + systemCPULoad, loadAvg, processStartTime, processOpenFDs) +} + +// recordMetrics record custom golang metrics +var recordMetrics = func(collectionInterval int32) { + for { + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfo.WithLabelValues(host.OS).Set(1) + availableCPUs.Set(float64(runtime.NumCPU())) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory.Set(float64(v.Free)) + usedVirtualMemory.Set(float64(v.Used)) + totalVirtualMemory.Set(float64(v.Total)) + + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad.Set(averagePercentage) + + pid := os.Getpid() + p, err := procfs.NewProc(pid) + if handleError(err, "Failed to get current process") { + return + } + stat, err := p.Stat() + if handleError(err, "Failed to get process stats") { + return + } + t, err := stat.StartTime() + if handleError(err, "Failed to read process start time") { + return + } + processStartTime.Set(t) + fds, err := p.FileDescriptorsLen() + if handleError(err, "Failed to read file descriptor count") { + return + } + processOpenFDs.Set(float64(fds)) + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + loadAvg.WithLabelValues("1m").Set(avg.Load1) + loadAvg.WithLabelValues("5m").Set(avg.Load5) + loadAvg.WithLabelValues("15m").Set(avg.Load15) + + // Sleep before the next measurement + time.Sleep(time.Duration(collectionInterval) * time.Second) + } + +} + +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) + return true + } + return false +} + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32, collectionInterval int32) { + done := make(chan struct{}) // Channel to indicate recordMetrics routine exit + + // Start the Prometheus metrics server + go func() { + http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + if err != nil { + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } + }() + + for { + // Start the recordMetrics goroutine + go func() { + defer func() { + if r := recover(); r != nil { + logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), + Severity: logging.MAJOR, + ErrorCode: 1111, + }) + } + }() + recordMetrics(collectionInterval) + done <- struct{}{} // Signal that the goroutine has completed + }() + + // Wait for the previous recordMetrics goroutine to complete + <-done + + // Log and restart the goroutine + logger.LoggerAPKOperator.Info("Restarting recordMetrics goroutine...") + time.Sleep(3 * time.Second) + } +} diff --git a/common-controller/pkg/metrics/types.go b/common-controller/pkg/metrics/types.go new file mode 100644 index 000000000..07761ebaf --- /dev/null +++ b/common-controller/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java index 9d751a267..42ae752ed 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java @@ -36,6 +36,8 @@ import org.wso2.apk.enforcer.discovery.service.subscription.JWTIssuerDiscoveryServiceGrpc; import org.wso2.apk.enforcer.discovery.subscription.JWTIssuer; import org.wso2.apk.enforcer.discovery.subscription.JWTIssuerList; +import org.wso2.apk.enforcer.jmx.JMXUtils; +import org.wso2.apk.enforcer.metrics.jmx.impl.ExtAuthMetrics; import org.wso2.apk.enforcer.subscription.SubscriptionDataHolder; import org.wso2.apk.enforcer.subscription.SubscriptionDataStore; import org.wso2.apk.enforcer.util.GRPCUtils; @@ -160,6 +162,10 @@ public void onNext(DiscoveryResponse response) { } subscriptionDataStore.addJWTIssuers(v); }); + + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordJWTIssuerMetrics(jwtIssuers.size()); + } logger.info("Number of jwt issuers received : " + jwtIssuers.size()); ack(); } catch (Exception e) { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index f815f0203..78cadd01e 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -66,7 +66,7 @@ public class ExtAuthService extends AuthorizationGrpc.AuthorizationImplBase { public void check(CheckRequest request, StreamObserver responseObserver) { TracingSpan extAuthServiceSpan = null; Scope extAuthServiceSpanScope = null; - long starTimestamp = System.currentTimeMillis(); + long startTimestamp = System.currentTimeMillis(); try { String traceId = request.getAttributes().getRequest().getHttp() .getHeadersOrDefault(HttpConstants.X_REQUEST_ID_HEADER, @@ -94,10 +94,10 @@ public void check(CheckRequest request, StreamObserver responseOb } if (MetricsManager.isMetricsEnabled()) { MetricsExporter metricsExporter = MetricsManager.getInstance(); - metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - starTimestamp); + metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - startTimestamp); } if (JMXUtils.isJMXMetricsEnabled()) { - ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - starTimestamp); + ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - startTimestamp); } } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java index 105eccf13..4755ebb45 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -42,7 +42,6 @@ public interface ExtAuthMetricsMXBean { */ public double getMaxResponseTimeMillis(); - /** * Getter for mimnimum response time in milliseconds. * @@ -51,7 +50,7 @@ public interface ExtAuthMetricsMXBean { public double getMinResponseTimeMillis(); /** - * Resets all the metrics to thier initial values. + * Resets all the metrics to their initial values. */ public void resetExtAuthMetrics(); @@ -69,4 +68,18 @@ public interface ExtAuthMetricsMXBean { */ public long getRequestCountWindowStartTimeMillis(); + /** + * Get the number of token issuers + * + * @return + */ + public int getTokenIssuerCount(); + + /** + * Get the number of subscriptions + * + * @return + */ + public int getSubscriptionCount(); + } \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index e01ac205c..29b68d653 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -29,10 +29,11 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; private static ExtAuthMetrics extAuthMetricsMBean = null; - private long requestCountInLastFiveMinuteWindow = 0; + private int jwtIssuerCount = 0; private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); private long totalRequestCount = 0; + private int subscriptionCount = 0; private double averageResponseTimeMillis = 0; private double maxResponseTimeMillis = Double.MIN_VALUE; private double minResponseTimeMillis = Double.MAX_VALUE; @@ -89,6 +90,14 @@ public synchronized void recordMetric(long responseTimeMillis) { this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); } + public synchronized void recordJWTIssuerMetrics(int jwtIssuers) { + this.jwtIssuerCount = jwtIssuers; + } + + public synchronized void recordSubscriptionMetrics(int subscriptionCount) { + this.subscriptionCount = subscriptionCount; + } + @Override public synchronized void resetExtAuthMetrics() { this.totalRequestCount = 0; @@ -112,4 +121,14 @@ public long getRequestCountInLastFiveMinuteWindow() { public long getRequestCountWindowStartTimeMillis() { return requestCountWindowStartTimeMillis; } + + @Override + public int getTokenIssuerCount() { + return jwtIssuerCount; + } + + @Override + public int getSubscriptionCount() { + return subscriptionCount; + } } \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java index 22a576d07..6c4905201 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java @@ -29,6 +29,8 @@ import org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping; import org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping; import org.wso2.apk.enforcer.discovery.subscription.Subscription; +import org.wso2.apk.enforcer.jmx.JMXUtils; +import org.wso2.apk.enforcer.metrics.jmx.impl.ExtAuthMetrics; import org.wso2.apk.enforcer.util.ApacheFeignHttpClient; import org.wso2.apk.enforcer.util.FilterUtils; @@ -157,6 +159,9 @@ private static void loadSubscriptions() { new Thread(() -> { SubscriptionListDto subscriptions = subscriptionValidationDataRetrievalRestClient.getAllSubscriptions(); List list = subscriptions.getList(); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(list.size()); + } Map> orgWizeMAp = new HashMap<>(); for (SubscriptionDto subscriptionDto : list) { String organization = subscriptionDto.getOrganization(); diff --git a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml index ef6c6c86e..5771c0027 100644 --- a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml +++ b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml @@ -21,12 +21,12 @@ lowercaseOutputName: true lowercaseOutputLabelNames: true rules: # WSO2 APK related metrics - - pattern: 'org.wso2.apk.enforcer<>total_request_count: (.*)' + - pattern: "org.wso2.apk.enforcer<>total_request_count: (.*)" name: org_wso2_apk_enforcer_request_count_total help: "WSO2 APK enforcer total request count." attrNameSnakeCase: true type: COUNTER - - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + - pattern: "org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis|token_issuer_count|subscription_count): (.*)" name: org_wso2_apk_enforcer_$1 help: "WSO2 APK enforcer $1." attrNameSnakeCase: true @@ -41,4 +41,4 @@ rules: name: os_$1 help: Operating System $1 attrNameSnakeCase: true - type: GAUGE \ No newline at end of file + type: GAUGE diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 89a53a89d..514790d41 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -26,6 +26,18 @@ data: level = "DEBUG" [ballerina.http] traceLogConsole = true + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [ballerina.observe] + metricsEnabled=true + metricsReporter="prometheus" + [ballerinax.prometheus] + port=18008 + {{if .Values.wso2.apk.metrics.configdeployerhost}} + host={{.Values.wso2.apk.metrics.configdeployerhost}} + {{else}} + host="127.0.0.1" + {{end}} + {{end}} [wso2.config_deployer_service.keyStores.tls] keyFilePath = "/home/wso2apk/config-deployer/security/config.key" certFilePath = "/home/wso2apk/config-deployer/security/config.pem" diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index 5a2647ca0..ef35c6424 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -48,6 +48,10 @@ spec: protocol: TCP - containerPort: 18003 protocol: TCP + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18009 + protocol: "TCP" + {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.commonController.deployment.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.commonController.deployment.env | indent 10 }} - name: OPERATOR_POD_NAMESPACE diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml index e5adf645f..debcb579f 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml @@ -38,4 +38,9 @@ spec: - name: https-internal-api protocol: TCP port: 18003 + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-common-controller-port + protocol: TCP + port: 18009 + {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 2a0a055d5..c9aeb8d4d 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -94,7 +94,7 @@ spec: {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config.yml {{- else }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} {{- if .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.redis }} - name: REDIS_USERNAME diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml index 71065b996..ce94cf2a4 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -13,7 +13,7 @@ data: help: "WSO2 APK enforcer total request count." attrNameSnakeCase: true type: COUNTER - - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis|token_issuer_count|subscription_count): (.*)' name: org_wso2_apk_enforcer_$1 help: "WSO2 APK enforcer $1." attrNameSnakeCase: true diff --git a/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal b/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal index 437a369e8..11907a88d 100644 --- a/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal +++ b/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal @@ -1,4 +1,6 @@ import ballerina/http; +import ballerinax/prometheus as _; + import wso2/apk_common_lib as commons; isolated service http:InterceptableService /api/deployer on ep0 { @@ -9,11 +11,11 @@ isolated service http:InterceptableService /api/deployer on ep0 { # anydata (API deployed successfully) # BadRequestError (Bad Request. Invalid request or validation error.) # InternalServerErrorError (Internal Server Error.) - isolated resource function post apis/deploy(http:RequestContext requestContext,http:Request request) returns commons:APKError|http:Response { + isolated resource function post apis/deploy(http:RequestContext requestContext, http:Request request) returns commons:APKError|http:Response { DeployerClient deployerClient = new; commons:UserContext authenticatedUserContext = check commons:getAuthenticatedUserContext(requestContext); commons:Organization organization = authenticatedUserContext.organization; - return check deployerClient.handleAPIDeployment(request,organization); + return check deployerClient.handleAPIDeployment(request, organization); } # Undeploy API # @@ -22,11 +24,11 @@ isolated service http:InterceptableService /api/deployer on ep0 { # AcceptedString (API undeployed successfully) # BadRequestError (Bad Request. Invalid request or validation error.) # InternalServerErrorError (Internal Server Error.) - isolated resource function post apis/undeploy(http:RequestContext requestContext,string apiId) returns AcceptedString|BadRequestError|InternalServerErrorError|commons:APKError { + isolated resource function post apis/undeploy(http:RequestContext requestContext, string apiId) returns AcceptedString|BadRequestError|InternalServerErrorError|commons:APKError { DeployerClient deployerClient = new; commons:UserContext authenticatedUserContext = check commons:getAuthenticatedUserContext(requestContext); commons:Organization organization = authenticatedUserContext.organization; - return check deployerClient.handleAPIUndeployment(apiId,organization); + return check deployerClient.handleAPIUndeployment(apiId, organization); } public function createInterceptors() returns http:Interceptor|http:Interceptor[] { From 847601be1cc80cc2ac5f96b2e86eb7c946f20fa5 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Sun, 25 Feb 2024 15:47:23 +0530 Subject: [PATCH 03/10] Update exposed adapter and enforcer metrics --- adapter/internal/discovery/xds/server.go | 34 +++++++++++++++++ .../operator/synchronizer/data_store.go | 5 +++ adapter/pkg/metrics/metrics.go | 37 ++++++++++++++++++- .../enforcer/constants/MetadataConstants.java | 2 +- .../apk/enforcer/grpc/ExtAuthService.java | 4 +- 5 files changed, 77 insertions(+), 5 deletions(-) diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index f85195bbf..da5dd9295 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -779,3 +779,37 @@ func SanitizeGateway(gatewayName string, create bool) error { } return nil } + +// GetEnvoyGatewayConfigClusters method gets the number of clusters in envoy gateway config +func GetEnvoyGatewayConfigClusters() int { + totalClusters := 0 + for _, config := range gatewayLabelConfigMap { + // Add the number of clusters in this EnvoyGatewayConfig instance to the total + totalClusters += len(config.clusters) + } + return totalClusters +} + +// GetEnvoyInternalAPIRoutes method gets the number of routes in envoy internal API +func GetEnvoyInternalAPIRoutes() int { + totalRoutes := 0 + for _, orgMap := range orgAPIMap { + for _, api := range orgMap { + // Add the number of routes in this EnvoyInternalAPI instance to the total + totalRoutes += len(api.routes) + } + } + return totalRoutes +} + +// GetEnvoyInternalAPIClusters method gets the number of clusters in envoy internal API +func GetEnvoyInternalAPIClusters() int { + totalClusters := 0 + for _, orgMap := range orgAPIMap { + for _, api := range orgMap { + // Add the number of clusters in this EnvoyInternalAPI instance to the total + totalClusters += len(api.clusters) + } + } + return totalClusters +} diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 80265707d..1ec062f86 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -571,3 +571,8 @@ func (ods *OperatorDataStore) DeleteCachedGateway(gatewayName types.NamespacedNa defer ods.mu.Unlock() delete(ods.gatewayStore, gatewayName) } + +// GetAPICount gets API count +func (ods *OperatorDataStore) GetAPICount() int { + return len(ods.apiStore) +} diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index 62a82e254..d62e21ef0 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -35,7 +35,9 @@ import ( "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/load" "github.com/shirou/gopsutil/v3/mem" + xds "github.com/wso2/apk/adapter/internal/discovery/xds" logger "github.com/wso2/apk/adapter/internal/loggers" + synchronizer "github.com/wso2/apk/adapter/internal/operator/synchronizer" "github.com/wso2/apk/adapter/pkg/logging" ) @@ -85,6 +87,26 @@ var ( Name: "process_open_fds", Help: "Number of open file descriptors.", }) + + apis = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "api_count", + Help: "Number of APIs created.", + }) + + internalClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "internal_cluster_count", + Help: "Number of internal clusters created.", + }) + + gwClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "gw_cluster_count", + Help: "Number of gw clusters created.", + }) + + internalRouteCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "internal_route_count", + Help: "Number of internal routes created.", + }) ) func init() { @@ -94,12 +116,23 @@ func init() { // Register other metrics prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs) + systemCPULoad, loadAvg, processStartTime, processOpenFDs, apis, internalClusterCount, internalRouteCount, gwClusterCount) } // recordMetrics record custom golang metrics var recordMetrics = func(collectionInterval int32) { for { + envoyInternalAPIClusterCount := xds.GetEnvoyInternalAPIClusters() + internalClusterCount.Set(float64(envoyInternalAPIClusterCount)) + + envoyInternalAPIRouteCount := xds.GetEnvoyInternalAPIRoutes() + internalRouteCount.Set(float64(envoyInternalAPIRouteCount)) + + gwClusterCount.Set(float64(xds.GetEnvoyGatewayConfigClusters())) + + apiCount := synchronizer.GetOperatorDataStore().GetAPICount() + apis.Set(float64(apiCount)) + host, err := host.Info() if handleError(err, "Failed to get host info") { return @@ -212,4 +245,4 @@ func StartPrometheusMetricsServer(port int32, collectionInterval int32) { logger.LoggerAPK.Info("Restarting recordMetrics goroutine...") time.Sleep(3 * time.Second) } -} \ No newline at end of file +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java index 191a7bcee..5dc541a2c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java @@ -53,7 +53,7 @@ public class MetadataConstants { public static final String CLIENT_IP_KEY = WSO2_METADATA_PREFIX + "client-ip"; public static final String ERROR_CODE_KEY = "ErrorCode"; - public static final String CHOREO_CONNECT_ENFORCER_REPLY = "apk-enforcer-reply"; + public static final String APK_ENFORCER_REPLY = "apk-enforcer-reply"; public static final String RATELIMIT_WSO2_ORG_PREFIX = "customorg"; public static final String GATEWAY_URL = WSO2_METADATA_PREFIX + "x-original-gw-url"; public static final String API_ENVIRONMENT = WSO2_METADATA_PREFIX + "api-environment"; diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index 78cadd01e..f0f30364e 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -142,7 +142,7 @@ private CheckResponse buildResponse(CheckRequest request, ResponseObject respons } addMetadata(metadataStructBuilder, "correlationID", responseObject.getCorrelationID()); - addMetadata(metadataStructBuilder, MetadataConstants.CHOREO_CONNECT_ENFORCER_REPLY, "Ok"); + addMetadata(metadataStructBuilder, MetadataConstants.APK_ENFORCER_REPLY, "Ok"); return checkResponseBuilder .setDynamicMetadata(metadataStructBuilder.build()) @@ -189,7 +189,7 @@ private CheckResponse buildResponse(CheckRequest request, ResponseObject respons responseObject.getRequestPath().split("\\?")[0]); // adding org level ratelimit key to metadata addMetadata(structBuilder, MetadataConstants.RATELIMIT_WSO2_ORG_PREFIX, responseObject.getOrganizationId()); - addMetadata(structBuilder, MetadataConstants.CHOREO_CONNECT_ENFORCER_REPLY, "Ok"); + addMetadata(structBuilder, MetadataConstants.APK_ENFORCER_REPLY, "Ok"); return CheckResponse.newBuilder().setStatus(Status.newBuilder().setCode(Code.OK_VALUE).build()) .setOkResponse(okResponseBuilder.build()) From 4502d72202e9466fd2cd84f577987666ce0d5dde Mon Sep 17 00:00:00 2001 From: sgayangi Date: Tue, 27 Feb 2024 14:55:46 +0530 Subject: [PATCH 04/10] Update enforcer metric configs --- .../apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java | 6 +++--- .../gateway-runtime/gateway-runtime-deployment.yaml | 8 ++++---- .../gateway-runtime/prometheus-jmx-configmap.yaml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index 29b68d653..be72384af 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -30,7 +30,7 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; private static ExtAuthMetrics extAuthMetricsMBean = null; private long requestCountInLastFiveMinuteWindow = 0; - private int jwtIssuerCount = 0; + private int tokenIssuerCount = 0; private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); private long totalRequestCount = 0; private int subscriptionCount = 0; @@ -91,7 +91,7 @@ public synchronized void recordMetric(long responseTimeMillis) { } public synchronized void recordJWTIssuerMetrics(int jwtIssuers) { - this.jwtIssuerCount = jwtIssuers; + this.tokenIssuerCount = jwtIssuers; } public synchronized void recordSubscriptionMetrics(int subscriptionCount) { @@ -124,7 +124,7 @@ public long getRequestCountWindowStartTimeMillis() { @Override public int getTokenIssuerCount() { - return jwtIssuerCount; + return tokenIssuerCount; } @Override diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index c9aeb8d4d..e9e696a1f 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -92,7 +92,7 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config.yml + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} @@ -163,8 +163,8 @@ spec: {{- end }} {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - name: prometheus-jmx-config-volume - mountPath: /tmp/metrics/prometheus-jmx-config.yml - subPath: prometheus-jmx-config.yml + mountPath: /tmp/metrics/prometheus-jmx-config-enforcer.yml + subPath: prometheus-jmx-config-enforcer.yml {{- end }} - name: enforcer-keystore-secret-volume mountPath: /home/wso2/security/keystore/enforcer.crt @@ -398,7 +398,7 @@ spec: {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - name: prometheus-jmx-config-volume configMap: - name: prometheus-jmx-config + name: prometheus-jmx-config-enforcer {{- end }} - name: enforcer-keystore-secret-volume secret: diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml index ce94cf2a4..dc1849b2d 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: prometheus-jmx-config + name: prometheus-jmx-config-enforcer data: - prometheus-jmx-config.yml: | + prometheus-jmx-config-enforcer.yml: | lowercaseOutputName: true lowercaseOutputLabelNames: true rules: From 8f77e25980d1a586212457c7ae54e01e9b33364a Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 28 Feb 2024 09:55:11 +0530 Subject: [PATCH 05/10] Add Prometheus support to Config DS --- .../config-deployer/config-ds-configmap.yaml | 8 ++-- .../config-deployer/config-ds-deployment.yaml | 12 +++++ .../config-ds-prometheus-jmx-configmap.yaml | 47 +++++++++++++++++++ .../config-deployer/config-ds-service.yaml | 8 ++++ helm-charts/values.yaml | 1 + .../ballerina/APIClient.bal | 5 +- .../ballerina/Ballerina.toml | 8 +++- .../ballerina/Dependencies.toml | 30 ++++++++++++ .../config-deployer-service/docker/Dockerfile | 2 + .../docker/config-deployer/config.sh | 13 ++--- .../config-deployer-service/java/build.gradle | 33 ++++++++++++- 11 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 514790d41..56da9842a 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -25,17 +25,17 @@ data: [ballerina.log] level = "DEBUG" [ballerina.http] - traceLogConsole = true + traceLogConsole = false {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} [ballerina.observe] metricsEnabled=true metricsReporter="prometheus" [ballerinax.prometheus] port=18008 - {{if .Values.wso2.apk.metrics.configdeployerhost}} - host={{.Values.wso2.apk.metrics.configdeployerhost}} + {{if .Values.wso2.apk.metrics.configDSBalPort}} + host="{{ .Values.wso2.apk.metrics.configDSBalPort}}" {{else}} - host="127.0.0.1" + host="0.0.0.0" {{end}} {{end}} [wso2.config_deployer_service.keyStores.tls] diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index 29d23a6df..acef54358 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -42,6 +42,12 @@ spec: ports: - containerPort: 9443 protocol: "TCP" + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18008 + protocol: "TCP" + - containerPort: 18010 + protocol: "TCP" + {{ end }} readinessProbe: httpGet: path: /health @@ -68,6 +74,9 @@ spec: {{- else }} subPath: tls.key {{- end }} + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml + subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.pem {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} @@ -105,6 +114,9 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-config-ds-configmap + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-configds - name: config-ds-tls-volume secret: {{ if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..610f358ec --- /dev/null +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-configds +data: + prometheus-jmx-config-configds.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # WSO2 APK related metrics + - pattern: 'org.wso2.apk.config<>total_request_count: (.*)' + name: org_wso2_apk_config_request_count_total + help: "WSO2 APK config total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.apk.config<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' + name: org_wso2_apk_config_$1 + help: "WSO2 APK config $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.apk.config<>(\w+): (.*)' + name: org_wso2_apk_config_thread_pool_$1 + help: "WSO2 APK config thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE \ No newline at end of file diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml index b19708a7f..ffd488cb4 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml @@ -28,4 +28,12 @@ spec: - name: servlet-https protocol: TCP port: 9443 + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-config-ds-port-lang + protocol: TCP + port: 18008 + - name: prometheus-config-ds-port-system + protocol: TCP + port: 18010 + {{ end }} {{- end -}} diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 00976c673..e670c336c 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -252,6 +252,7 @@ wso2: metrics: enabled: true + # configDSBalPort: 0.0.0.0 idp: enabled: true listener: diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index d6441725f..c0009d947 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -27,10 +27,9 @@ import ballerina/uuid; import ballerina/crypto; import config_deployer_service.java.io as javaio; import wso2/apk_common_lib as commons; - -# +import ballerinax/prometheus as _; public class APIClient { - + # This function used to convert APKInternalAPI model to APKConf. # # + api - APKInternalAPI model diff --git a/runtime/config-deployer-service/ballerina/Ballerina.toml b/runtime/config-deployer-service/ballerina/Ballerina.toml index 46bc216ad..82f0b4f11 100644 --- a/runtime/config-deployer-service/ballerina/Ballerina.toml +++ b/runtime/config-deployer-service/ballerina/Ballerina.toml @@ -5,7 +5,7 @@ version = "1.1.0-SNAPSHOT" distribution = "2201.8.0" [build-options] -observabilityIncluded = false +observabilityIncluded = true [[dependency]] org = "wso2" @@ -49,6 +49,12 @@ groupId = "io.swagger" artifactId = "swagger-core" version = "1.6.10" +# transitive dependency of io.prometheus.jmx:jmx_prometheus_javaagent:0.20.0 +[[platform.java11.dependency]] +groupId = "io.prometheus.jmx" +artifactId = "jmx_prometheus_javaagent" +version = "0.20.0" + # transitive dependency of io.swagger.parser.v3:swagger-parser:2.1.13 [[platform.java11.dependency]] groupId = "io.swagger" diff --git a/runtime/config-deployer-service/ballerina/Dependencies.toml b/runtime/config-deployer-service/ballerina/Dependencies.toml index 9c45a8850..bb1ba5577 100644 --- a/runtime/config-deployer-service/ballerina/Dependencies.toml +++ b/runtime/config-deployer-service/ballerina/Dependencies.toml @@ -365,6 +365,18 @@ modules = [ {org = "ballerina", packageName = "uuid", moduleName = "uuid"} ] +[[package]] +org = "ballerinai" +name = "observe" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "observe"} +] +modules = [ + {org = "ballerinai", packageName = "observe", moduleName = "observe"} +] + [[package]] org = "ballerinax" name = "postgresql" @@ -377,6 +389,21 @@ dependencies = [ {org = "ballerina", name = "time"} ] +[[package]] +org = "ballerinax" +name = "prometheus" +version = "0.2.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "regex"} +] +modules = [ + {org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"} +] + [[package]] org = "wso2" name = "apk_common_lib" @@ -420,6 +447,8 @@ dependencies = [ {org = "ballerina", name = "test"}, {org = "ballerina", name = "url"}, {org = "ballerina", name = "uuid"}, + {org = "ballerinai", name = "observe"}, + {org = "ballerinax", name = "prometheus"}, {org = "wso2", name = "apk_common_lib"} ] modules = [ @@ -432,6 +461,7 @@ modules = [ {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.model"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.api"}, + {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.definitions"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.model"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.partitionClient"} ] diff --git a/runtime/config-deployer-service/docker/Dockerfile b/runtime/config-deployer-service/docker/Dockerfile index 82f8ba2aa..a7df67708 100644 --- a/runtime/config-deployer-service/docker/Dockerfile +++ b/runtime/config-deployer-service/docker/Dockerfile @@ -53,6 +53,8 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \ ADD config-deployer ${USER_HOME}/config-deployer COPY docker-entrypoint.sh ${USER_HOME} +COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/ + RUN chown -R ${USER} ${USER_HOME}/config-deployer \ && chown ${USER} /home/${USER}/docker-entrypoint.sh \ && chgrp -R 0 ${USER_HOME} \ diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index cfb35a976..acac82f1b 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -98,9 +98,10 @@ fi echo "Using Java memory options: $JVM_MEM_OPTS" $JAVACMD \ - $JVM_MEM_OPTS \ - $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ - -jar config_deployer_service.jar $* - status=$? \ No newline at end of file + $JVM_MEM_OPTS \ + $JAVA_OPTS \ + -classpath "$CLASSPATH" \ + -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ + -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18010:/tmp/metrics/prometheus-jmx-config-configds.yml \ + -jar config_deployer_service.jar $* +status=$? diff --git a/runtime/config-deployer-service/java/build.gradle b/runtime/config-deployer-service/java/build.gradle index 95e278c66..a7f868785 100644 --- a/runtime/config-deployer-service/java/build.gradle +++ b/runtime/config-deployer-service/java/build.gradle @@ -17,8 +17,6 @@ apply from: "$projectDir/../../../common-gradle-scripts/java.gradle" - - dependencies { implementation libs.commons.logging implementation libs.gson @@ -34,4 +32,35 @@ dependencies { implementation libs.handy.uri.templates implementation libs.json implementation libs.commons.lang + implementation libs.prometheus +} + +tasks.register('copy_dependencies', Copy) { + dependsOn create_lib + from configurations.default + into "$projectDir/../docker/lib" + finalizedBy build +} + +tasks.register('create_lib') { + dependsOn delete_lib + finalizedBy copy_dependencies } + +tasks.register('delete_lib', Delete) { + delete "$projectDir/lib" + finalizedBy create_lib +} + +tasks.named('build').configure { + dependsOn copy_dependencies +} + +configurations.implementation { + exclude group: 'jline' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.apache.logging' + exclude group: 'log4j' + exclude group: 'org.ops4j.pax.logging' + exclude group: 'com.google.guava' +} \ No newline at end of file From be8647a0ce7d79f501b93a429a34dd198c0677f9 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 28 Feb 2024 12:33:16 +0530 Subject: [PATCH 06/10] Update configurations for Prometheus --- .../internal/config/default_config.go | 2 +- .../config-deployer/config-ds-deployment.yaml | 12 ++--- .../config-ds-prometheus-jmx-configmap.yaml | 18 +------ .../config-deployer/config-ds-service.yaml | 8 --- .../adapter/adapter-service.yaml | 5 -- .../common-controller-deployment.yaml | 2 +- .../common-controller-service.yaml | 5 -- .../gateway-runtime/enforcer-service.yaml | 5 -- .../prometheus-jmx-configmap.yaml | 16 ++++++ .../gateway-runtime/router-service.yaml | 31 ----------- .../docker/config-deployer/config.sh | 53 ++++++++++--------- 11 files changed, 52 insertions(+), 105 deletions(-) delete mode 100644 helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 2391f9524..269b19bd4 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -46,7 +46,7 @@ var defaultConfig = &Config{ Metrics: metrics{ Enabled: true, Type: "prometheus", - Port: 18009, + Port: 18010, CollectionInterval: 5, }, }, diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index acef54358..e4d520600 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -45,7 +45,7 @@ spec: {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - containerPort: 18008 protocol: "TCP" - - containerPort: 18010 + - containerPort: 18009 protocol: "TCP" {{ end }} readinessProbe: @@ -67,6 +67,9 @@ spec: - mountPath: /home/wso2apk/config-deployer/conf/Config.toml name: config-toml-volume subPath: Config.toml + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml + subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.key {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} @@ -74,9 +77,6 @@ spec: {{- else }} subPath: tls.key {{- end }} - - name: prometheus-jmx-config-volume - mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml - subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.pem {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} @@ -114,8 +114,8 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-config-ds-configmap - - name: prometheus-jmx-config-volume - configMap: + - name: prometheus-jmx-config-volume + configMap: name: prometheus-jmx-config-configds - name: config-ds-tls-volume secret: diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml index 610f358ec..de2731309 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml @@ -23,25 +23,9 @@ data: lowercaseOutputName: true lowercaseOutputLabelNames: true rules: - # WSO2 APK related metrics - - pattern: 'org.wso2.apk.config<>total_request_count: (.*)' - name: org_wso2_apk_config_request_count_total - help: "WSO2 APK config total request count." - attrNameSnakeCase: true - type: COUNTER - - pattern: 'org.wso2.apk.config<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis): (.*)' - name: org_wso2_apk_config_$1 - help: "WSO2 APK config $1." - attrNameSnakeCase: true - type: GAUGE - - pattern: 'org.wso2.apk.config<>(\w+): (.*)' - name: org_wso2_apk_config_thread_pool_$1 - help: "WSO2 APK config thread pool $1." - attrNameSnakeCase: true - type: GAUGE # OS related metrics - pattern: 'java.lang<>(\w+): (.*)' name: os_$1 help: Operating System $1 attrNameSnakeCase: true - type: GAUGE \ No newline at end of file + type: GAUGE diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml index ffd488cb4..b19708a7f 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-service.yaml @@ -28,12 +28,4 @@ spec: - name: servlet-https protocol: TCP port: 9443 - {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - name: prometheus-config-ds-port-lang - protocol: TCP - port: 18008 - - name: prometheus-config-ds-port-system - protocol: TCP - port: 18010 - {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml index c0c8e0c7c..8c78ff45c 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-service.yaml @@ -41,9 +41,4 @@ spec: - name: profile-port protocol: TCP port: 6060 - {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} - - name: prometheus-adapter-port - port: 18006 - protocol: "TCP" - {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index ef35c6424..133cd8441 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -49,7 +49,7 @@ spec: - containerPort: 18003 protocol: TCP {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - containerPort: 18009 + - containerPort: 18010 protocol: "TCP" {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.commonController.deployment.resources | indent 10 }} diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml index debcb579f..e5adf645f 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-service.yaml @@ -38,9 +38,4 @@ spec: - name: https-internal-api protocol: TCP port: 18003 - {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - name: prometheus-common-controller-port - protocol: TCP - port: 18009 - {{ end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml index a8355ce99..e88b6d861 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/enforcer-service.yaml @@ -28,9 +28,4 @@ spec: - name: servlet-https protocol: TCP port: 9092 - {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - name: prometheus-enforcer-port - protocol: TCP - port: 18007 - {{- end }} {{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml index dc1849b2d..4f45e2d4f 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -1,3 +1,19 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml deleted file mode 100644 index 8450a504d..000000000 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/router-service.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. -# -# WSO2 LLC. licenses this file to you under the Apache License, -# Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{{- if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "apk-helm.resource.prefix" . }}-router-service - namespace : {{ .Release.Namespace }} -spec: - # label keys and values that must match in order to receive traffic for this service - selector: -{{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "gateway" ) | indent 4}} - ports: - - name: prometheus-router-port - protocol: TCP - port: 9000 -{{- end -}} diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index acac82f1b..9cc37c02d 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -2,23 +2,26 @@ PRG="$0" while [ -h "$PRG" ]; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then + ls=$(ls -ld "$PRG") + link=$(expr "$ls" : '.*-> \(.*\)$') + if expr "$link" : '.*/.*' >/dev/null; then PRG="$link" else - PRG=`dirname "$PRG"`/"$link" + PRG=$(dirname "$PRG")/"$link" fi done # Get standard environment variables -PRGDIR=`dirname "$PRG"` +PRGDIR=$(dirname "$PRG") -[ -z "$RUNTIME_HOME" ] && RUNTIME_HOME=`cd "$PRGDIR" ; pwd` +[ -z "$RUNTIME_HOME" ] && RUNTIME_HOME=$( + cd "$PRGDIR" + pwd +) -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then +if [ -z "$JAVACMD" ]; then + if [ -n "$JAVA_HOME" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else @@ -29,7 +32,7 @@ if [ -z "$JAVACMD" ] ; then fi fi -if [ ! -x "$JAVACMD" ] ; then +if [ ! -x "$JAVACMD" ]; then echo "Error: JAVA_HOME is not defined correctly." echo " Admin cannot execute $JAVACMD" exit 1 @@ -42,16 +45,15 @@ if [ -z "$JAVA_HOME" ]; then fi # ----- Process the input command ---------------------------------------------- args="" -for c in $* -do - if [ "$c" = "--debug" ] || [ "$c" = "-debug" ] || [ "$c" = "debug" ]; then - CMD="--debug" - continue - elif [ "$CMD" = "--debug" ]; then - if [ -z "$PORT" ]; then - PORT=$c - fi +for c in $*; do + if [ "$c" = "--debug" ] || [ "$c" = "-debug" ] || [ "$c" = "debug" ]; then + CMD="--debug" + continue + elif [ "$CMD" = "--debug" ]; then + if [ -z "$PORT" ]; then + PORT=$c fi + fi done if [ "$CMD" = "--debug" ]; then @@ -69,11 +71,10 @@ fi CLASSPATH="" if [ -e "$JAVA_HOME/lib/tools.jar" ]; then - CLASSPATH="$JAVA_HOME/lib/tools.jar" + CLASSPATH="$JAVA_HOME/lib/tools.jar" fi -for t in "$RUNTIME_HOME"/lib/*.jar -do - CLASSPATH="$CLASSPATH":$t +for t in "$RUNTIME_HOME"/lib/*.jar; do + CLASSPATH="$CLASSPATH":$t done # ----- Execute The Requested Command ----------------------------------------- @@ -85,15 +86,15 @@ cd "$RUNTIME_HOME" TMP_DIR="$RUNTIME_HOME"/tmp if [ -d "$TMP_DIR" ]; then -rm -rf "$TMP_DIR"/* + rm -rf "$TMP_DIR"/* fi START_EXIT_STATUS=121 status=$START_EXIT_STATUS if [ -z "$JVM_MEM_OPTS" ]; then - java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') - JVM_MEM_OPTS="-Xms256m -Xmx1024m" + java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') + JVM_MEM_OPTS="-Xms256m -Xmx1024m" fi echo "Using Java memory options: $JVM_MEM_OPTS" From 2273c652a0f8e23b0d3824a62a6676d41eeb9d25 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 29 Feb 2024 11:54:47 +0530 Subject: [PATCH 07/10] Add Prometheus support to IdP DS --- .../config-deployer/config-ds-configmap.yaml | 4 +-- .../idp/idp-ds/idp-ds-configmap.yaml | 14 ++++++++- .../idp/idp-ds/idp-ds-deployment.yaml | 12 +++++++ .../idp-ds-prometheus-jmx-configmap.yaml | 31 +++++++++++++++++++ helm-charts/values.yaml | 6 ++-- helm-charts/values.yaml.template | 8 +++++ .../ballerina/Ballerina.toml | 5 +++ .../ballerina/Dependencies.toml | 18 ++++++++++- idp/idp-domain-service/ballerina/Init.bal | 1 + idp/idp-domain-service/docker/Dockerfile | 1 + idp/idp-domain-service/docker/build.gradle | 29 +++++++++++++++-- idp/idp-domain-service/docker/idp/idp.sh | 13 ++++---- .../docker/config-deployer/config.sh | 2 +- 13 files changed, 129 insertions(+), 15 deletions(-) create mode 100644 helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 56da9842a..df984d60c 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -32,8 +32,8 @@ data: metricsReporter="prometheus" [ballerinax.prometheus] port=18008 - {{if .Values.wso2.apk.metrics.configDSBalPort}} - host="{{ .Values.wso2.apk.metrics.configDSBalPort}}" + {{if .Values.wso2.apk.metrics.configDSBalHost}} + host="{{ .Values.wso2.apk.metrics.configDSBalHost}}" {{else}} host="0.0.0.0" {{end}} diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml index 2abf4dcf8..b98a26f77 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml @@ -25,7 +25,19 @@ data: [ballerina.log] level = "DEBUG" [ballerina.http] - traceLogConsole = true + traceLogConsole = false + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [ballerina.observe] + metricsEnabled=true + metricsReporter="prometheus" + [ballerinax.prometheus] + port=18011 + {{if .Values.wso2.apk.metrics.idpDSBalHost}} + host="{{ .Values.wso2.apk.metrics.idpDSBalHost}}" + {{else}} + host="0.0.0.0" + {{end}} + {{end}} # Sample configurations [wso2.idp_domain_service.datasourceConfiguration] description = "Database for idp" diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml index d246dd99d..079209c48 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml @@ -51,6 +51,12 @@ spec: ports: - containerPort: 9443 protocol: "TCP" + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18011 + protocol: "TCP" + - containerPort: 18012 + protocol: "TCP" + {{ end }} readinessProbe: httpGet: path: /health @@ -76,6 +82,9 @@ spec: - mountPath: /home/wso2apk/idp/security/wso2carbon.pem name: idp-signing-keystore-volume subPath: wso2carbon.pem + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-idpds.yml + subPath: prometheus-jmx-config-idpds.yml - name: idp-ds-tls-volume mountPath: /home/wso2apk/idp/security/idp.key {{- if and .Values.idp.idpds.configs .Values.idp.idpds.configs.tls }} @@ -111,6 +120,9 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-idp-ds-configmap + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-idpds - name: idp-signing-keystore-volume secret: secretName: {{ template "apk-helm.resource.prefix" . }}-idp-signing-keystore-secret diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..773615b87 --- /dev/null +++ b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-idpds +data: + prometheus-jmx-config-idpds.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index e670c336c..88a34f604 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -251,8 +251,10 @@ wso2: # hostnameVerifier: "AllowAll" metrics: - enabled: true - # configDSBalPort: 0.0.0.0 + enabled: false + # configDSBalHost: 0.0.0.0 + # idpDSBalHost: 0.0.0.0 + idp: enabled: true listener: diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 0c04fc44c..ebfa2189a 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -592,6 +592,14 @@ wso2: keepAliveTime: 600 # -- Queue size of the worker threads queueSize: 1000 + metrics: + # -- Enable Prometheus metrics + enabled: false + # -- Configure the host for exposing the config ds ballerina metrics + # configDSBalHost: 0.0.0.0 + # -- Configure the host for exposing the idp ds ballerina metrics. + # idpDSBalHost: 0.0.0.0 + idp: # -- Enable Non production identity server enabled: true diff --git a/idp/idp-domain-service/ballerina/Ballerina.toml b/idp/idp-domain-service/ballerina/Ballerina.toml index 7f9e44b8e..ac2a6ec1e 100644 --- a/idp/idp-domain-service/ballerina/Ballerina.toml +++ b/idp/idp-domain-service/ballerina/Ballerina.toml @@ -9,5 +9,10 @@ groupId = "org.postgresql" artifactId = "postgresql" version = "42.2.20" +[[platform.java11.dependency]] +groupId = "io.prometheus.jmx" +artifactId = "jmx_prometheus_javaagent" +version = "0.20.0" + [build-options] observabilityIncluded = true diff --git a/idp/idp-domain-service/ballerina/Dependencies.toml b/idp/idp-domain-service/ballerina/Dependencies.toml index fc72aa1d3..210772f7e 100644 --- a/idp/idp-domain-service/ballerina/Dependencies.toml +++ b/idp/idp-domain-service/ballerina/Dependencies.toml @@ -375,6 +375,21 @@ modules = [ {org = "ballerinax", packageName = "postgresql", moduleName = "postgresql"} ] +[[package]] +org = "ballerinax" +name = "prometheus" +version = "0.2.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "regex"} +] +modules = [ + {org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"} +] + [[package]] org = "wso2" name = "idp_domain_service" @@ -392,7 +407,8 @@ dependencies = [ {org = "ballerina", name = "url"}, {org = "ballerina", name = "uuid"}, {org = "ballerinai", name = "observe"}, - {org = "ballerinax", name = "postgresql"} + {org = "ballerinax", name = "postgresql"}, + {org = "ballerinax", name = "prometheus"} ] modules = [ {org = "wso2", packageName = "idp_domain_service", moduleName = "idp_domain_service"} diff --git a/idp/idp-domain-service/ballerina/Init.bal b/idp/idp-domain-service/ballerina/Init.bal index b34ebe42a..78fe6721e 100644 --- a/idp/idp-domain-service/ballerina/Init.bal +++ b/idp/idp-domain-service/ballerina/Init.bal @@ -20,6 +20,7 @@ import ballerina/log; import ballerina/sql; import ballerina/http; import ballerina/jwt; +import ballerinax/prometheus as _; configurable IDPConfiguration & readonly idpConfiguration = ?; final postgresql:Client|sql:Error dbClient; diff --git a/idp/idp-domain-service/docker/Dockerfile b/idp/idp-domain-service/docker/Dockerfile index 6f8406c82..f07cf7498 100644 --- a/idp/idp-domain-service/docker/Dockerfile +++ b/idp/idp-domain-service/docker/Dockerfile @@ -72,6 +72,7 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \ ADD idp ${USER_HOME}/idp COPY docker-entrypoint.sh ${USER_HOME} +COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/ RUN chown -R ${USER} ${USER_HOME}/idp \ && chown ${USER} /home/${USER}/docker-entrypoint.sh \ diff --git a/idp/idp-domain-service/docker/build.gradle b/idp/idp-domain-service/docker/build.gradle index 75f812f75..03c2890c1 100644 --- a/idp/idp-domain-service/docker/build.gradle +++ b/idp/idp-domain-service/docker/build.gradle @@ -18,20 +18,45 @@ apply from: "$projectDir/../../../common-gradle-scripts/docker.gradle" apply from: "$rootDir/../../common-gradle-scripts/copy.gradle" +apply from: "$projectDir/../../../common-gradle-scripts/java.gradle" +dependencies { + implementation libs.prometheus +} + +tasks.register('copy_dependencies', Copy) { + dependsOn create_lib + from configurations.default + into "$projectDir/../docker/lib" + finalizedBy build +} + +tasks.register('create_lib') { + dependsOn delete_lib + finalizedBy copy_dependencies +} + +tasks.register('delete_lib', Delete) { + delete "$projectDir/lib" + finalizedBy create_lib +} + +tasks.named('build').configure { + dependsOn copy_dependencies +} tasks.named('copy_dist').configure{ finalizedBy docker_build } -task build{ +task buildDockerImage{ group 'build' description 'Build docker image' dependsOn 'copy_dist' dependsOn 'docker_build' } -build.configure{ +buildDockerImage.configure{ mustRunAfter(":ballerina:build") dependsOn(":ballerina:build") } \ No newline at end of file diff --git a/idp/idp-domain-service/docker/idp/idp.sh b/idp/idp-domain-service/docker/idp/idp.sh index ffa63a2c1..03cb90968 100755 --- a/idp/idp-domain-service/docker/idp/idp.sh +++ b/idp/idp-domain-service/docker/idp/idp.sh @@ -98,9 +98,10 @@ fi echo "Using Java memory options: $JVM_MEM_OPTS" $JAVACMD \ - $JVM_MEM_OPTS \ - $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Djava.io.tmpdir="$IDP_HOME/tmp" \ - -jar idp_domain_service.jar $* - status=$? \ No newline at end of file + $JVM_MEM_OPTS \ + $JAVA_OPTS \ + -classpath "$CLASSPATH" \ + -Djava.io.tmpdir="$IDP_HOME/tmp" \ + -javaagent:/home/wso2apk/jmx_prometheus_javaagent-0.20.0.jar=18012:/tmp/metrics/prometheus-jmx-config-idpds.yml \ + -jar idp_domain_service.jar $* +status=$? diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index 9cc37c02d..ccc254e6a 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -103,6 +103,6 @@ $JAVACMD \ $JAVA_OPTS \ -classpath "$CLASSPATH" \ -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ - -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18010:/tmp/metrics/prometheus-jmx-config-configds.yml \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18009:/tmp/metrics/prometheus-jmx-config-configds.yml \ -jar config_deployer_service.jar $* status=$? From 7764bcc606937b8f3a52f6d996a1eb06847b10e5 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Fri, 1 Mar 2024 15:55:13 +0530 Subject: [PATCH 08/10] Add Prometheus support to ratelimiter --- .../ratelimiter/ratelimiter-deployment.yaml | 16 +++++++++++++++- helm-charts/values.yaml | 12 ++++++++++++ helm-charts/values.yaml.template | 13 +++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 7f15be10f..5821cb0c1 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -35,6 +35,14 @@ spec: automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }} + - name: statsd + image: {{ .Values.wso2.apk.metrics.statsd.image.repository }}:{{ .Values.wso2.apk.metrics.statsd.image.tag }} + imagePullPolicy: {{ .Values.wso2.apk.metrics.statsd.imagePullPolicy }} + ports: + - name: statsd-metrics + containerPort: 9102 + {{ end }} - name: ratelimiter image: {{ .Values.wso2.apk.dp.ratelimiter.deployment.image }} imagePullPolicy: {{ .Values.wso2.apk.dp.ratelimiter.deployment.imagePullPolicy }} @@ -52,7 +60,13 @@ spec: - name: PORT value: "8090" - name: USE_STATSD - value: "false" + value: "{{ and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }}" + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }} + - name: STATSD_HOST + value: "0.0.0.0" + - name: STATSD_PORT + value: "9125" + {{- end}} - name: LOG_LEVEL value: "DEBUG" - name: LIMIT_RESPONSE_HEADERS_ENABLED diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 88a34f604..4f709376a 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -254,6 +254,18 @@ wso2: enabled: false # configDSBalHost: 0.0.0.0 # idpDSBalHost: 0.0.0.0 + # statsd: + # enabled: true + # image: + # repository: prom/statsd-exporter + # tag: v0.26.0 + # imagePullPolicy: IfNotPresent + # resources: + # limits: + # memory: 128Mi + # requests: + # cpu: 0.1 + # memory: 64Mi idp: enabled: true diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index ebfa2189a..9c0f1418c 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -599,6 +599,19 @@ wso2: # configDSBalHost: 0.0.0.0 # -- Configure the host for exposing the idp ds ballerina metrics. # idpDSBalHost: 0.0.0.0 + # -- Statsd is required to expose metrics from ratelimiter + # statsd: + # enabled: true + # image: + # repository: prom/statsd-exporter + # tag: v0.26.0 + # imagePullPolicy: IfNotPresent + # resources: + # limits: + # memory: 128Mi + # requests: + # cpu: 0.1 + # memory: 64Mi idp: # -- Enable Non production identity server From 180e5c5679da5a2f4ff0d1e5cd2758f4b64cf10f Mon Sep 17 00:00:00 2001 From: sgayangi Date: Sun, 3 Mar 2024 09:49:46 +0530 Subject: [PATCH 09/10] Address review comments --- adapter/config/default_config.go | 7 +- adapter/go.mod | 8 +- adapter/go.sum | 13 +- adapter/internal/adapter/adapter.go | 2 +- adapter/internal/discovery/xds/server.go | 11 + .../operator/synchronizer/data_store.go | 5 - adapter/pkg/metrics/metrics.go | 256 ++++-------------- .../commoncontroller/common_controller.go | 6 +- common-controller/go.mod | 8 +- common-controller/go.sum | 9 +- .../internal/config/default_config.go | 7 +- common-controller/pkg/metrics/metrics.go | 191 +------------ common-go-libs/go.mod | 15 +- common-go-libs/go.sum | 27 +- common-go-libs/pkg/metrics/metrics.go | 179 ++++++++++++ common-go-libs/pkg/metrics/types.go | 24 ++ .../discovery/JWTIssuerDiscoveryClient.java | 3 +- .../subscription/SubscriptionDataHolder.java | 23 +- .../subscription/SubscriptionDataStore.java | 12 +- .../SubscriptionDataStoreImpl.java | 13 + .../SubscriptionDataStoreUtil.java | 9 +- .../gateway-components/common-log-conf.yaml | 6 + .../gateway-components/log-conf.yaml | 6 + .../ratelimiter/ratelimiter-deployment.yaml | 11 +- helm-charts/values.yaml | 1 - helm-charts/values.yaml.template | 1 - 26 files changed, 420 insertions(+), 433 deletions(-) create mode 100644 common-go-libs/pkg/metrics/metrics.go create mode 100644 common-go-libs/pkg/metrics/types.go diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 1fe3847d7..72dd7cd01 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -33,10 +33,9 @@ var defaultConfig = &Config{ }, Environment: "Default", Metrics: metrics{ - Enabled: true, - Type: "prometheus", - Port: 18006, - CollectionInterval: 5, + Enabled: false, + Type: "prometheus", + Port: 18006, }, }, Envoy: envoy{ diff --git a/adapter/go.mod b/adapter/go.mod index 03faae8b7..de51aaa18 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -10,9 +10,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/pelletier/go-toml v1.9.5 - github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 - github.com/shirou/gopsutil/v3 v3.24.1 + github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb @@ -62,12 +60,14 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/vektah/gqlparser v1.3.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.21.0 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index 0af7a43be..b59fd5b9a 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -117,8 +117,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -128,8 +128,8 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -159,8 +159,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -214,7 +214,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 83575ee23..ec8d8eb3e 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -160,7 +160,7 @@ func Run(conf *config.Config) { // Start the metrics server if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { logger.LoggerAPK.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port, conf.Adapter.Metrics.CollectionInterval) + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) } cache := xds.GetXdsCache() diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index da5dd9295..cd7607a78 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -813,3 +813,14 @@ func GetEnvoyInternalAPIClusters() int { } return totalClusters } + +// GetEnvoyInternalAPICount method gets the number of APIs deployed in Envoy +func GetEnvoyInternalAPICount() int { + totalCount := 0 + for _, apiMap := range orgAPIMap { + for range apiMap { + totalCount++ + } + } + return totalCount +} diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 1ec062f86..80265707d 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -571,8 +571,3 @@ func (ods *OperatorDataStore) DeleteCachedGateway(gatewayName types.NamespacedNa defer ods.mu.Unlock() delete(ods.gatewayStore, gatewayName) } - -// GetAPICount gets API count -func (ods *OperatorDataStore) GetAPICount() int { - return len(ods.apiStore) -} diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go index d62e21ef0..d0f797471 100644 --- a/adapter/pkg/metrics/metrics.go +++ b/adapter/pkg/metrics/metrics.go @@ -22,227 +22,89 @@ package metrics import ( "fmt" "net/http" - "os" - "runtime" "strconv" - "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/procfs" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/load" - "github.com/shirou/gopsutil/v3/mem" xds "github.com/wso2/apk/adapter/internal/discovery/xds" logger "github.com/wso2/apk/adapter/internal/loggers" - synchronizer "github.com/wso2/apk/adapter/internal/operator/synchronizer" "github.com/wso2/apk/adapter/pkg/logging" + commonmetrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) var ( prometheusMetricRegistry = prometheus.NewRegistry() - - hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", - }, []string{"os"}) - - availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", - }) - - freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory.", - }) - - totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_total_virtual_memory_bytes", - Help: "Amount of total virtual memory.", - }) - usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_used_virtual_memory_bytes", - Help: "Amount of used virtual memory.", - }) - - systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", - }) - - loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", - }, []string{"duration"}) - - processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_start_time_seconds", - Help: "Start time of the process since unix epoch in seconds.", - }) - - processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_open_fds", - Help: "Number of open file descriptors.", - }) - - apis = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "api_count", - Help: "Number of APIs created.", - }) - - internalClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "internal_cluster_count", - Help: "Number of internal clusters created.", - }) - - gwClusterCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "gw_cluster_count", - Help: "Number of gw clusters created.", - }) - - internalRouteCount = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "internal_route_count", - Help: "Number of internal routes created.", - }) ) -func init() { - // Register the Go collector with the registry - goCollector := collectors.NewGoCollector() - prometheusMetricRegistry.MustRegister(goCollector) - - // Register other metrics - prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs, apis, internalClusterCount, internalRouteCount, gwClusterCount) +// AdapterCollector contains the descriptions of the custom metrics exposed by the adapter. +// It also uses the metrics defined in common-go-libs +type AdapterCollector struct { + commonmetrics.Collector + apis *prometheus.Desc + internalClusterCount *prometheus.Desc + internalRouteCount *prometheus.Desc } -// recordMetrics record custom golang metrics -var recordMetrics = func(collectionInterval int32) { - for { - envoyInternalAPIClusterCount := xds.GetEnvoyInternalAPIClusters() - internalClusterCount.Set(float64(envoyInternalAPIClusterCount)) - - envoyInternalAPIRouteCount := xds.GetEnvoyInternalAPIRoutes() - internalRouteCount.Set(float64(envoyInternalAPIRouteCount)) - - gwClusterCount.Set(float64(xds.GetEnvoyGatewayConfigClusters())) - - apiCount := synchronizer.GetOperatorDataStore().GetAPICount() - apis.Set(float64(apiCount)) - - host, err := host.Info() - if handleError(err, "Failed to get host info") { - return - } - hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) - - v, err := mem.VirtualMemory() - if handleError(err, "Failed to read virtual memory metrics") { - return - } - freePhysicalMemory.Set(float64(v.Free)) - usedVirtualMemory.Set(float64(v.Used)) - totalVirtualMemory.Set(float64(v.Total)) +func adapterMetricsCollector() *AdapterCollector { + return &AdapterCollector{ + Collector: *commonmetrics.CustomMetricsCollector(), + apis: prometheus.NewDesc( + "api_count", + "Number of APIs created.", + nil, nil, + ), + internalClusterCount: prometheus.NewDesc( + "internal_cluster_count", + "Number of internal clusters created.", + nil, nil, + ), + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + } +} - percentages, err := cpu.Percent(0, false) - if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { - return - } - totalPercentage := 0.0 - for _, p := range percentages { - totalPercentage += p - } - averagePercentage := totalPercentage / float64(len(percentages)) - systemCPULoad.Set(averagePercentage) +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *AdapterCollector) Describe(ch chan<- *prometheus.Desc) { + collector.Collector.Describe(ch) + ch <- collector.apis + ch <- collector.internalClusterCount + ch <- collector.internalRouteCount +} - pid := os.Getpid() - p, err := procfs.NewProc(pid) - if handleError(err, "Failed to get current process") { - return - } - stat, err := p.Stat() - if handleError(err, "Failed to get process stats") { - return - } - t, err := stat.StartTime() - if handleError(err, "Failed to read process start time") { - return - } - processStartTime.Set(t) - fds, err := p.FileDescriptorsLen() - if handleError(err, "Failed to read file descriptor count") { - return - } - processOpenFDs.Set(float64(fds)) +// Collect collects all the relevant Prometheus metrics. +func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) { + collector.Collector.Collect(ch) + var apisCount float64 + var internalClusterCount float64 + var internalRouteCount float64 - avg, err := load.Avg() - if handleError(err, "Failed to read cpu load averages") { - return - } - loadAvg.WithLabelValues("1m").Set(avg.Load1) - loadAvg.WithLabelValues("5m").Set(avg.Load5) - loadAvg.WithLabelValues("15m").Set(avg.Load15) + apiCount := xds.GetEnvoyInternalAPICount() + apisCount = float64(apiCount) - // Sleep before the next measurement - time.Sleep(time.Duration(collectionInterval) * time.Second) - } + internalRouteCount = float64(xds.GetEnvoyInternalAPIRoutes()) + internalClusterCount = float64(xds.GetEnvoyInternalAPIClusters()) + ch <- prometheus.MustNewConstMetric(collector.apis, prometheus.GaugeValue, apisCount) + ch <- prometheus.MustNewConstMetric(collector.internalRouteCount, prometheus.GaugeValue, internalRouteCount) + ch <- prometheus.MustNewConstMetric(collector.internalClusterCount, prometheus.GaugeValue, internalClusterCount) } -func handleError(err error, message string) bool { +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { + + collector := adapterMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintf(message, err.Error()), - Severity: logging.MINOR, - ErrorCode: 1109, + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, }) - return true - } - return false -} - -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32, collectionInterval int32) { - done := make(chan struct{}) // Channel to indicate recordMetrics routine exit - - // Start the Prometheus metrics server - go func() { - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) - if err != nil { - logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, - }) - } - }() - - for { - // Start the recordMetrics goroutine - go func() { - defer func() { - if r := recover(); r != nil { - logger.LoggerAPK.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), - Severity: logging.MAJOR, - ErrorCode: 1111, - }) - } - }() - recordMetrics(collectionInterval) - done <- struct{}{} // Signal that the goroutine has completed - }() - - // Wait for the previous recordMetrics goroutine to complete - <-done - - // Log and restart the goroutine - logger.LoggerAPK.Info("Restarting recordMetrics goroutine...") - time.Sleep(3 * time.Second) } } diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index 676c87bff..d270085b2 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -206,14 +206,14 @@ func InitCommonControllerServer(conf *config.Config) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - loggers.LoggerAPKOperator.Info("Starting common controller ....") - // Start the metrics server if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") - go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port, conf.CommonController.Metrics.CollectionInterval) + go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port) } + loggers.LoggerAPKOperator.Info("Starting common controller ....") + rateLimiterCache := xds.GetRateLimiterCache() rlsSrv := xdsv3.NewServer(ctx, rateLimiterCache, &xds.Callbacks{}) diff --git a/common-controller/go.mod b/common-controller/go.mod index 9013a7ac6..fa2172190 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -6,7 +6,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/procfs v0.12.0 + github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -20,9 +20,8 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 - github.com/shirou/gopsutil/v3 v3.24.1 github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 - github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 + github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 google.golang.org/grpc v1.62.0 ) @@ -51,13 +50,14 @@ require ( github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vektah/gqlparser v1.3.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index b678ced88..b98da4f27 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -186,8 +186,8 @@ github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= -github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -304,7 +304,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 269b19bd4..396ec2b16 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -44,10 +44,9 @@ var defaultConfig = &Config{ Persistence: persistence{Type: "K8s"}, }, Metrics: metrics{ - Enabled: true, - Type: "prometheus", - Port: 18010, - CollectionInterval: 5, + Enabled: false, + Type: "prometheus", + Port: 18010, }, }, } diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go index ef7824ebb..c2da1fa33 100644 --- a/common-controller/pkg/metrics/metrics.go +++ b/common-controller/pkg/metrics/metrics.go @@ -16,200 +16,33 @@ * under the License. */ -// Package metrics holds the implementation for exposing common controller metrics to prometheus +// Package metrics holds the implementation for exposing adapter metrics to prometheus package metrics import ( "fmt" "net/http" - "os" - "runtime" "strconv" - "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/prometheus/procfs" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/load" - "github.com/shirou/gopsutil/v3/mem" "github.com/wso2/apk/adapter/pkg/logging" logger "github.com/wso2/apk/common-controller/internal/loggers" + metrics "github.com/wso2/apk/common-go-libs/pkg/metrics" ) -var ( - prometheusMetricRegistry = prometheus.NewRegistry() - - hostInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "host_info", - Help: "Host Info", - }, []string{"os"}) - - availableCPUs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_available_cpu_total", - Help: "Number of available CPUs.", - }) - - freePhysicalMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_free_physical_memory_bytes", - Help: "Amount of free physical memory.", - }) - - totalVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_total_virtual_memory_bytes", - Help: "Amount of total virtual memory.", - }) - usedVirtualMemory = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_used_virtual_memory_bytes", - Help: "Amount of used virtual memory.", - }) - - systemCPULoad = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "os_system_cpu_load_percentage", - Help: "System-wide CPU usage as a percentage.", - }) - - loadAvg = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "os_system_load_average", - Help: "Current load of CPU in the host system for the last {x} minutes", - }, []string{"duration"}) - - processStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_start_time_seconds", - Help: "Start time of the process since unix epoch in seconds.", - }) - - processOpenFDs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "process_open_fds", - Help: "Number of open file descriptors.", - }) -) - -func init() { - // Register the Go collector with the registry - goCollector := collectors.NewGoCollector() - prometheusMetricRegistry.MustRegister(goCollector) - - // Register other metrics - prometheusMetricRegistry.MustRegister(hostInfo, availableCPUs, freePhysicalMemory, usedVirtualMemory, totalVirtualMemory, - systemCPULoad, loadAvg, processStartTime, processOpenFDs) -} - -// recordMetrics record custom golang metrics -var recordMetrics = func(collectionInterval int32) { - for { - host, err := host.Info() - if handleError(err, "Failed to get host info") { - return - } - hostInfo.WithLabelValues(host.OS).Set(1) - availableCPUs.Set(float64(runtime.NumCPU())) - - v, err := mem.VirtualMemory() - if handleError(err, "Failed to read virtual memory metrics") { - return - } - freePhysicalMemory.Set(float64(v.Free)) - usedVirtualMemory.Set(float64(v.Used)) - totalVirtualMemory.Set(float64(v.Total)) - - percentages, err := cpu.Percent(0, false) - if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { - return - } - totalPercentage := 0.0 - for _, p := range percentages { - totalPercentage += p - } - averagePercentage := totalPercentage / float64(len(percentages)) - systemCPULoad.Set(averagePercentage) - - pid := os.Getpid() - p, err := procfs.NewProc(pid) - if handleError(err, "Failed to get current process") { - return - } - stat, err := p.Stat() - if handleError(err, "Failed to get process stats") { - return - } - t, err := stat.StartTime() - if handleError(err, "Failed to read process start time") { - return - } - processStartTime.Set(t) - fds, err := p.FileDescriptorsLen() - if handleError(err, "Failed to read file descriptor count") { - return - } - processOpenFDs.Set(float64(fds)) - - avg, err := load.Avg() - if handleError(err, "Failed to read cpu load averages") { - return - } - loadAvg.WithLabelValues("1m").Set(avg.Load1) - loadAvg.WithLabelValues("5m").Set(avg.Load5) - loadAvg.WithLabelValues("15m").Set(avg.Load15) - - // Sleep before the next measurement - time.Sleep(time.Duration(collectionInterval) * time.Second) - } - -} +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { -func handleError(err error, message string) bool { + collector := metrics.CustomMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintf(message, err.Error()), - Severity: logging.MINOR, - ErrorCode: 1109, + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, }) - return true - } - return false -} - -// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. -func StartPrometheusMetricsServer(port int32, collectionInterval int32) { - done := make(chan struct{}) // Channel to indicate recordMetrics routine exit - - // Start the Prometheus metrics server - go func() { - http.Handle("/metrics", promhttp.HandlerFor(prometheusMetricRegistry, promhttp.HandlerOpts{})) - err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) - if err != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("Prometheus metrics server error:", err), - Severity: logging.MAJOR, - ErrorCode: 1110, - }) - } - }() - - for { - // Start the recordMetrics goroutine - go func() { - defer func() { - if r := recover(); r != nil { - logger.LoggerAPKOperator.ErrorC(logging.ErrorDetails{ - Message: fmt.Sprintln("recordMetrics goroutine exited with error:", r), - Severity: logging.MAJOR, - ErrorCode: 1111, - }) - } - }() - recordMetrics(collectionInterval) - done <- struct{}{} // Signal that the goroutine has completed - }() - - // Wait for the previous recordMetrics goroutine to complete - <-done - - // Log and restart the goroutine - logger.LoggerAPKOperator.Info("Restarting recordMetrics goroutine...") - time.Sleep(3 * time.Second) } } diff --git a/common-go-libs/go.mod b/common-go-libs/go.mod index 37ab6563a..fcb0a6169 100644 --- a/common-go-libs/go.mod +++ b/common-go-libs/go.mod @@ -21,6 +21,16 @@ require ( sigs.k8s.io/gateway-api v0.7.1 ) +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect +) + require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -55,18 +65,19 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/common-go-libs/go.sum b/common-go-libs/go.sum index e4cdd32d2..b7ff16a46 100644 --- a/common-go-libs/go.sum +++ b/common-go-libs/go.sum @@ -43,6 +43,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -69,6 +71,7 @@ github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvR github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -95,6 +98,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -117,6 +122,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -131,6 +138,12 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -147,6 +160,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU= github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 h1:l+OdeDCNWPgie7L1fCjpfH04mAL3rFi4U+/idE8eduA= @@ -155,6 +172,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -206,17 +225,21 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= diff --git a/common-go-libs/pkg/metrics/metrics.go b/common-go-libs/pkg/metrics/metrics.go new file mode 100644 index 000000000..40abd594e --- /dev/null +++ b/common-go-libs/pkg/metrics/metrics.go @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "runtime" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/wso2/apk/adapter/pkg/logging" + logger "github.com/wso2/apk/common-go-libs/loggers" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() +) + +// Collector contains the descriptions of the custom metrics exposed +type Collector struct { + internalRouteCount *prometheus.Desc + hostInfo *prometheus.Desc + availableCPUs *prometheus.Desc + freePhysicalMemory *prometheus.Desc + totalVirtualMemory *prometheus.Desc + usedVirtualMemory *prometheus.Desc + systemCPULoad *prometheus.Desc + loadAvg *prometheus.Desc +} + +// CustomMetricsCollector contains the descriptions of the custom metrics exposed +func CustomMetricsCollector() *Collector { + return &Collector{ + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + hostInfo: prometheus.NewDesc( + "host_info", + "Host Info", + []string{"os"}, nil, + ), + availableCPUs: prometheus.NewDesc( + "os_available_cpu_total", + "Number of available CPUs.", + nil, nil, + ), + freePhysicalMemory: prometheus.NewDesc( + "os_free_physical_memory_bytes", + "Amount of free physical memory.", + nil, nil, + ), + totalVirtualMemory: prometheus.NewDesc( + "os_total_virtual_memory_bytes", + "Amount of total virtual memory.", + nil, nil, + ), + usedVirtualMemory: prometheus.NewDesc( + "os_used_virtual_memory_bytes", + "Amount of used virtual memory.", + nil, nil, + ), + systemCPULoad: prometheus.NewDesc( + "os_system_cpu_load_percentage", + "System-wide CPU usage as a percentage.", + nil, nil, + ), + loadAvg: prometheus.NewDesc( + "os_system_load_average", + "Current load of CPU in the host system for the last {x} minutes", + []string{"duration"}, nil, + ), + } +} + +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *Collector) Describe(ch chan<- *prometheus.Desc) { + ch <- collector.internalRouteCount + ch <- collector.hostInfo + ch <- collector.availableCPUs + ch <- collector.freePhysicalMemory + ch <- collector.totalVirtualMemory + ch <- collector.usedVirtualMemory + ch <- collector.systemCPULoad + ch <- collector.loadAvg +} + +// Collect collects all the relevant Prometheus metrics. +func (collector *Collector) Collect(ch chan<- prometheus.Metric) { + + var hostInfoValue float64 + var availableCPUs float64 + var freePhysicalMemory float64 + var totalVirtualMemory float64 + var usedVirtualMemory float64 + var systemCPULoad float64 + + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfoValue = 1 + availableCPUs = float64(runtime.NumCPU()) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory = float64(v.Free) + usedVirtualMemory = float64(v.Used) + totalVirtualMemory = float64(v.Total) + + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad = averagePercentage + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + + ch <- prometheus.MustNewConstMetric(collector.hostInfo, prometheus.GaugeValue, hostInfoValue, host.OS) + ch <- prometheus.MustNewConstMetric(collector.availableCPUs, prometheus.GaugeValue, availableCPUs) + ch <- prometheus.MustNewConstMetric(collector.freePhysicalMemory, prometheus.GaugeValue, freePhysicalMemory) + ch <- prometheus.MustNewConstMetric(collector.usedVirtualMemory, prometheus.GaugeValue, usedVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.totalVirtualMemory, prometheus.GaugeValue, totalVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.systemCPULoad, prometheus.GaugeValue, systemCPULoad) + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load1, "1m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load5, "5m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load15, "15m") +} + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) +} + +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) + return true + } + return false +} diff --git a/common-go-libs/pkg/metrics/types.go b/common-go-libs/pkg/metrics/types.go new file mode 100644 index 000000000..07761ebaf --- /dev/null +++ b/common-go-libs/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java index 42ae752ed..e4311e4c9 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java @@ -162,9 +162,8 @@ public void onNext(DiscoveryResponse response) { } subscriptionDataStore.addJWTIssuers(v); }); - if (JMXUtils.isJMXMetricsEnabled()) { - ExtAuthMetrics.getInstance().recordJWTIssuerMetrics(jwtIssuers.size()); + ExtAuthMetrics.getInstance().recordJWTIssuerMetrics(SubscriptionDataHolder.getInstance().getTotalJWTIssuerCount()); } logger.info("Number of jwt issuers received : " + jwtIssuers.size()); ack(); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java index 4454ef588..c90bdcf14 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java @@ -27,7 +27,8 @@ public class SubscriptionDataHolder { private static final SubscriptionDataHolder instance = new SubscriptionDataHolder(); - Map subscriptionDataStoreMap = new HashMap<>(); + Map subscriptionDataStoreMap = new HashMap<>(); + public static SubscriptionDataHolder getInstance() { return instance; } @@ -35,9 +36,27 @@ public static SubscriptionDataHolder getInstance() { public SubscriptionDataStore getSubscriptionDataStore(String organization) { return subscriptionDataStoreMap.get(organization); } + public SubscriptionDataStore initializeSubscriptionDataStore(String organization) { SubscriptionDataStore subscriptionDataStore = new SubscriptionDataStoreImpl(); - subscriptionDataStoreMap.put(organization,subscriptionDataStore); + subscriptionDataStoreMap.put(organization, subscriptionDataStore); return subscriptionDataStore; } + + public int getTotalSubscriptionCount() { + int totalSubCount = 0; + for (SubscriptionDataStore store : subscriptionDataStoreMap.values()) { + totalSubCount += store.getSubscriptionCount(); + } + return totalSubCount; + } + + public int getTotalJWTIssuerCount() { + int totalJWTIssuerCount = 0; + for (SubscriptionDataStore store : subscriptionDataStoreMap.values()) { + totalJWTIssuerCount += store.getJWTIssuerCount(); + } + return totalJWTIssuerCount; + } + } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java index 33be66ea2..3fca55d2d 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java @@ -51,7 +51,6 @@ public interface SubscriptionDataStore { void addApplications(List applicationList); - void addApplicationKeyMappings( List applicationKeyMappingList); @@ -73,7 +72,7 @@ void addApplicationKeyMappings( * @return ApplicationKeyMapping which match the given parameters */ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentifier, String keyType, - String securityScheme, String envType); + String securityScheme, String envType); /** * Filter the applications map based on the provided parameters. @@ -108,11 +107,13 @@ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentif void addApplicationMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping applicationMapping); - void addApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); + void addApplicationKeyMapping( + org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); void removeApplicationMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping applicationMapping); - void removeApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); + void removeApplicationKeyMapping( + org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); void removeSubscription(org.wso2.apk.enforcer.discovery.subscription.Subscription subscription); @@ -120,4 +121,7 @@ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentif public void addApplicationMappings(List applicationMappingList); + int getSubscriptionCount(); + + int getJWTIssuerCount(); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java index 33c132e65..04eaf413c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java @@ -370,6 +370,7 @@ public void removeApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscrip public void removeSubscription(org.wso2.apk.enforcer.discovery.subscription.Subscription subscription) { subscriptionMap.remove(subscription.getUuid()); + } @Override @@ -398,4 +399,16 @@ private String getMapKey(String environment, String issuer) { return environment + DELEM_PERIOD + issuer; } + @Override + public int getSubscriptionCount() { + + return subscriptionMap.size(); + } + + @Override + public int getJWTIssuerCount() { + + return jwtValidatorMap.size(); + } + } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java index 6c4905201..de2aa497d 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java @@ -160,7 +160,7 @@ private static void loadSubscriptions() { SubscriptionListDto subscriptions = subscriptionValidationDataRetrievalRestClient.getAllSubscriptions(); List list = subscriptions.getList(); if (JMXUtils.isJMXMetricsEnabled()) { - ExtAuthMetrics.getInstance().recordSubscriptionMetrics(list.size()); + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); } Map> orgWizeMAp = new HashMap<>(); for (SubscriptionDto subscriptionDto : list) { @@ -193,6 +193,9 @@ public static void addSubscription(Subscription subscription) { SubscriptionDataStore subscriptionDataStore = getSubscriptionDataStore(subscription.getOrganization()); subscriptionDataStore.addSubscription(subscription); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); + } } @@ -245,8 +248,10 @@ public static void removeApplicationKeyMapping(ApplicationKeyMapping application public static void removeSubscription(Subscription subscription) { SubscriptionDataStore subscriptionDataStore = getSubscriptionDataStore(subscription.getOrganization()); - subscriptionDataStore.removeSubscription(subscription); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); + } } diff --git a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml index 0fa0bb360..768f3fd45 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml @@ -10,6 +10,12 @@ data: {{- if and .Values.wso2.apk.dp.environment .Values.wso2.apk.dp.environment.name }} environment = "{{ .Values.wso2.apk.dp.environment.name }}" {{- end }} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [commoncontroller.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18010 + {{ end}} [commoncontroller.server] label = "ratelimiter" {{ if and .Values.wso2.apk.dp.commonController.configs .Values.wso2.apk.dp.commonController.configs.apiNamespaces }} diff --git a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml index 7b775168b..2fc6b24fd 100644 --- a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml @@ -14,6 +14,12 @@ data: [adapter.operator] namespaces = [{{ include "commaJoinedQuotedList" .Values.wso2.apk.dp.adapter.configs.apiNamespaces}}] {{ end}} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [adapter.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18006 + {{ end}} {{ if and .Values.wso2.apk.dp.gatewayRuntime.deployment .Values.wso2.apk.dp.gatewayRuntime.deployment.router .Values.wso2.apk.dp.gatewayRuntime.deployment.router.configs }} [router] diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 5821cb0c1..a56500cd5 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -35,7 +35,7 @@ spec: automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: - {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }} + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd }} - name: statsd image: {{ .Values.wso2.apk.metrics.statsd.image.repository }}:{{ .Values.wso2.apk.metrics.statsd.image.tag }} imagePullPolicy: {{ .Values.wso2.apk.metrics.statsd.imagePullPolicy }} @@ -59,14 +59,17 @@ spec: value: "0.0.0.0" - name: PORT value: "8090" + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd}} - name: USE_STATSD - value: "{{ and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }}" - {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd.enabled }} + value: "true" - name: STATSD_HOST value: "0.0.0.0" - name: STATSD_PORT value: "9125" - {{- end}} + {{ else }} + - name: USE_STATSD + value: "false" + {{ end }} - name: LOG_LEVEL value: "DEBUG" - name: LIMIT_RESPONSE_HEADERS_ENABLED diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 4f709376a..bbc801126 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -255,7 +255,6 @@ wso2: # configDSBalHost: 0.0.0.0 # idpDSBalHost: 0.0.0.0 # statsd: - # enabled: true # image: # repository: prom/statsd-exporter # tag: v0.26.0 diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 9c0f1418c..550602951 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -601,7 +601,6 @@ wso2: # idpDSBalHost: 0.0.0.0 # -- Statsd is required to expose metrics from ratelimiter # statsd: - # enabled: true # image: # repository: prom/statsd-exporter # tag: v0.26.0 From 85bcb6a83af518754f8861c342b3d68adfe0354b Mon Sep 17 00:00:00 2001 From: sgayangi Date: Wed, 6 Mar 2024 15:29:57 +0530 Subject: [PATCH 10/10] Update ports used for Prometheus --- common-controller/internal/config/default_config.go | 2 +- .../data-plane/config-deployer/config-ds-configmap.yaml | 2 +- .../data-plane/config-deployer/config-ds-deployment.yaml | 4 ++-- .../common-controller/common-controller-deployment.yaml | 2 +- .../data-plane/gateway-components/common-log-conf.yaml | 2 +- .../gateway-runtime/gateway-runtime-deployment.yaml | 4 ++-- helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml | 2 +- helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml | 4 ++-- idp/idp-domain-service/build.gradle | 2 +- idp/idp-domain-service/docker/build.gradle | 2 +- idp/idp-domain-service/docker/idp/idp.sh | 2 +- .../config-deployer-service/docker/config-deployer/config.sh | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 396ec2b16..40971fea6 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -46,7 +46,7 @@ var defaultConfig = &Config{ Metrics: metrics{ Enabled: false, Type: "prometheus", - Port: 18010, + Port: 18006, }, }, } diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index df984d60c..6a4e56650 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -31,7 +31,7 @@ data: metricsEnabled=true metricsReporter="prometheus" [ballerinax.prometheus] - port=18008 + port=18006 {{if .Values.wso2.apk.metrics.configDSBalHost}} host="{{ .Values.wso2.apk.metrics.configDSBalHost}}" {{else}} diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index e4d520600..27d71680b 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -43,9 +43,9 @@ spec: - containerPort: 9443 protocol: "TCP" {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - containerPort: 18008 + - containerPort: 18006 protocol: "TCP" - - containerPort: 18009 + - containerPort: 18007 protocol: "TCP" {{ end }} readinessProbe: diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index 133cd8441..eead36fcc 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -49,7 +49,7 @@ spec: - containerPort: 18003 protocol: TCP {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - containerPort: 18010 + - containerPort: 18006 protocol: "TCP" {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.commonController.deployment.resources | indent 10 }} diff --git a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml index 768f3fd45..12777a55d 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml @@ -14,7 +14,7 @@ data: [commoncontroller.metrics] enabled = {{.Values.wso2.apk.metrics.enabled}} type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" - port = 18010 + port = 18006 {{ end}} [commoncontroller.server] label = "ratelimiter" diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index e9e696a1f..bf2883c1e 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -53,7 +53,7 @@ spec: - containerPort: 18002 protocol: "TCP" {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} - - containerPort: 18007 + - containerPort: 18006 protocol: "TCP" {{- end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.resources | indent 10 }} @@ -92,7 +92,7 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-enforcer.yml + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml index b98a26f77..62c690912 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml @@ -31,7 +31,7 @@ data: metricsEnabled=true metricsReporter="prometheus" [ballerinax.prometheus] - port=18011 + port=18006 {{if .Values.wso2.apk.metrics.idpDSBalHost}} host="{{ .Values.wso2.apk.metrics.idpDSBalHost}}" {{else}} diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml index 079209c48..466b4da4a 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml @@ -52,9 +52,9 @@ spec: - containerPort: 9443 protocol: "TCP" {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - - containerPort: 18011 + - containerPort: 18006 protocol: "TCP" - - containerPort: 18012 + - containerPort: 18007 protocol: "TCP" {{ end }} readinessProbe: diff --git a/idp/idp-domain-service/build.gradle b/idp/idp-domain-service/build.gradle index 612190f25..cfe38cb8e 100644 --- a/idp/idp-domain-service/build.gradle +++ b/idp/idp-domain-service/build.gradle @@ -36,7 +36,7 @@ release { unSnapshotVersion.finalizedBy ":ballerina:commit_toml_files" afterReleaseBuild.dependsOn ":docker:docker_push" task build{ - dependsOn("docker:build") + dependsOn("docker:buildDockerImage") } build.configure { dependsOn(":docker:build") diff --git a/idp/idp-domain-service/docker/build.gradle b/idp/idp-domain-service/docker/build.gradle index 03c2890c1..25d8b11f0 100644 --- a/idp/idp-domain-service/docker/build.gradle +++ b/idp/idp-domain-service/docker/build.gradle @@ -59,4 +59,4 @@ task buildDockerImage{ buildDockerImage.configure{ mustRunAfter(":ballerina:build") dependsOn(":ballerina:build") - } \ No newline at end of file +} \ No newline at end of file diff --git a/idp/idp-domain-service/docker/idp/idp.sh b/idp/idp-domain-service/docker/idp/idp.sh index 03cb90968..dc8c6e8bc 100755 --- a/idp/idp-domain-service/docker/idp/idp.sh +++ b/idp/idp-domain-service/docker/idp/idp.sh @@ -102,6 +102,6 @@ $JAVACMD \ $JAVA_OPTS \ -classpath "$CLASSPATH" \ -Djava.io.tmpdir="$IDP_HOME/tmp" \ - -javaagent:/home/wso2apk/jmx_prometheus_javaagent-0.20.0.jar=18012:/tmp/metrics/prometheus-jmx-config-idpds.yml \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-idpds.yml \ -jar idp_domain_service.jar $* status=$? diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index ccc254e6a..3d9253f7e 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -103,6 +103,6 @@ $JAVACMD \ $JAVA_OPTS \ -classpath "$CLASSPATH" \ -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ - -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18009:/tmp/metrics/prometheus-jmx-config-configds.yml \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-configds.yml \ -jar config_deployer_service.jar $* status=$?