diff --git a/adapter/config/default_config.go b/adapter/config/default_config.go index 86630ad57..72dd7cd01 100644 --- a/adapter/config/default_config.go +++ b/adapter/config/default_config.go @@ -32,6 +32,11 @@ var defaultConfig = &Config{ Namespaces: nil, }, Environment: "Default", + Metrics: metrics{ + Enabled: false, + Type: "prometheus", + Port: 18006, + }, }, Envoy: envoy{ ListenerCodecType: "AUTO", diff --git a/adapter/config/types.go b/adapter/config/types.go index f51f2577f..8e8954f38 100644 --- a/adapter/config/types.go +++ b/adapter/config/types.go @@ -93,6 +93,8 @@ type adapter struct { Operator operator // Environment of the Adapter Environment string + // Metric represents configurations to expose/export go metrics + Metrics metrics } // Envoy Listener Component related configurations. @@ -299,8 +301,10 @@ type tracing struct { } type metrics struct { - Enabled bool - Type string + Enabled bool + Type string + Port int32 + CollectionInterval int32 } type analyticsAdapter struct { diff --git a/adapter/go.mod b/adapter/go.mod index b276c876e..de51aaa18 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/pelletier/go-toml v1.9.5 + github.com/prometheus/client_golang v1.17.0 github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb @@ -34,6 +35,7 @@ require ( github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -47,6 +49,7 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -54,12 +57,17 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/vektah/gqlparser v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.21.0 // indirect diff --git a/adapter/go.sum b/adapter/go.sum index 26378461c..b59fd5b9a 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -41,6 +41,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -62,6 +64,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -88,6 +91,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -110,6 +115,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= @@ -121,6 +128,12 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -136,12 +149,18 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU= github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -182,15 +201,19 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/adapter/internal/adapter/adapter.go b/adapter/internal/adapter/adapter.go index 08ba6d809..ec8d8eb3e 100644 --- a/adapter/internal/adapter/adapter.go +++ b/adapter/internal/adapter/adapter.go @@ -20,6 +20,7 @@ package adapter import ( "crypto/tls" + "strings" "time" discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" @@ -33,6 +34,7 @@ import ( wso2_server "github.com/wso2/apk/adapter/pkg/discovery/protocol/server/v3" "github.com/wso2/apk/adapter/pkg/health" healthservice "github.com/wso2/apk/adapter/pkg/health/api/wso2/health/service" + metrics "github.com/wso2/apk/adapter/pkg/metrics" "github.com/wso2/apk/adapter/pkg/utils/tlsutils" "context" @@ -154,6 +156,13 @@ func Run(conf *config.Config) { } logger.LoggerAPK.Info("Starting adapter ....") + + // Start the metrics server + if conf.Adapter.Metrics.Enabled && strings.EqualFold(conf.Adapter.Metrics.Type, metrics.PrometheusMetricType) { + logger.LoggerAPK.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.Adapter.Metrics.Port) + } + cache := xds.GetXdsCache() enforcerCache := xds.GetEnforcerCache() enforcerAPICache := xds.GetEnforcerAPICache() diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index f85195bbf..cd7607a78 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -779,3 +779,48 @@ func SanitizeGateway(gatewayName string, create bool) error { } return nil } + +// GetEnvoyGatewayConfigClusters method gets the number of clusters in envoy gateway config +func GetEnvoyGatewayConfigClusters() int { + totalClusters := 0 + for _, config := range gatewayLabelConfigMap { + // Add the number of clusters in this EnvoyGatewayConfig instance to the total + totalClusters += len(config.clusters) + } + return totalClusters +} + +// GetEnvoyInternalAPIRoutes method gets the number of routes in envoy internal API +func GetEnvoyInternalAPIRoutes() int { + totalRoutes := 0 + for _, orgMap := range orgAPIMap { + for _, api := range orgMap { + // Add the number of routes in this EnvoyInternalAPI instance to the total + totalRoutes += len(api.routes) + } + } + return totalRoutes +} + +// GetEnvoyInternalAPIClusters method gets the number of clusters in envoy internal API +func GetEnvoyInternalAPIClusters() int { + totalClusters := 0 + for _, orgMap := range orgAPIMap { + for _, api := range orgMap { + // Add the number of clusters in this EnvoyInternalAPI instance to the total + totalClusters += len(api.clusters) + } + } + return totalClusters +} + +// GetEnvoyInternalAPICount method gets the number of APIs deployed in Envoy +func GetEnvoyInternalAPICount() int { + totalCount := 0 + for _, apiMap := range orgAPIMap { + for range apiMap { + totalCount++ + } + } + return totalCount +} diff --git a/adapter/pkg/metrics/metrics.go b/adapter/pkg/metrics/metrics.go new file mode 100644 index 000000000..d0f797471 --- /dev/null +++ b/adapter/pkg/metrics/metrics.go @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + xds "github.com/wso2/apk/adapter/internal/discovery/xds" + logger "github.com/wso2/apk/adapter/internal/loggers" + "github.com/wso2/apk/adapter/pkg/logging" + commonmetrics "github.com/wso2/apk/common-go-libs/pkg/metrics" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() +) + +// AdapterCollector contains the descriptions of the custom metrics exposed by the adapter. +// It also uses the metrics defined in common-go-libs +type AdapterCollector struct { + commonmetrics.Collector + apis *prometheus.Desc + internalClusterCount *prometheus.Desc + internalRouteCount *prometheus.Desc +} + +func adapterMetricsCollector() *AdapterCollector { + return &AdapterCollector{ + Collector: *commonmetrics.CustomMetricsCollector(), + apis: prometheus.NewDesc( + "api_count", + "Number of APIs created.", + nil, nil, + ), + internalClusterCount: prometheus.NewDesc( + "internal_cluster_count", + "Number of internal clusters created.", + nil, nil, + ), + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + } +} + +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *AdapterCollector) Describe(ch chan<- *prometheus.Desc) { + collector.Collector.Describe(ch) + ch <- collector.apis + ch <- collector.internalClusterCount + ch <- collector.internalRouteCount +} + +// Collect collects all the relevant Prometheus metrics. +func (collector *AdapterCollector) Collect(ch chan<- prometheus.Metric) { + collector.Collector.Collect(ch) + var apisCount float64 + var internalClusterCount float64 + var internalRouteCount float64 + + apiCount := xds.GetEnvoyInternalAPICount() + apisCount = float64(apiCount) + + internalRouteCount = float64(xds.GetEnvoyInternalAPIRoutes()) + internalClusterCount = float64(xds.GetEnvoyInternalAPIClusters()) + + ch <- prometheus.MustNewConstMetric(collector.apis, prometheus.GaugeValue, apisCount) + ch <- prometheus.MustNewConstMetric(collector.internalRouteCount, prometheus.GaugeValue, internalRouteCount) + ch <- prometheus.MustNewConstMetric(collector.internalClusterCount, prometheus.GaugeValue, internalClusterCount) +} + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { + + collector := adapterMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } +} diff --git a/adapter/pkg/metrics/types.go b/adapter/pkg/metrics/types.go new file mode 100644 index 000000000..8e9fae144 --- /dev/null +++ b/adapter/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package metrics + + const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" + ) \ No newline at end of file diff --git a/common-controller/commoncontroller/common_controller.go b/common-controller/commoncontroller/common_controller.go index ea071545c..d270085b2 100644 --- a/common-controller/commoncontroller/common_controller.go +++ b/common-controller/commoncontroller/common_controller.go @@ -25,6 +25,7 @@ import ( "net" "os" "os/signal" + "strings" "time" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -40,6 +41,7 @@ import ( "github.com/wso2/apk/common-controller/internal/server" utils "github.com/wso2/apk/common-controller/internal/utils" xds "github.com/wso2/apk/common-controller/internal/xds" + "github.com/wso2/apk/common-controller/pkg/metrics" apkmgt "github.com/wso2/apk/common-go-libs/pkg/discovery/api/wso2/discovery/service/apkmgt" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -204,6 +206,12 @@ func InitCommonControllerServer(conf *config.Config) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Start the metrics server + if conf.CommonController.Metrics.Enabled && strings.EqualFold(conf.CommonController.Metrics.Type, metrics.PrometheusMetricType) { + loggers.LoggerAPKOperator.Info("Starting Prometheus Metrics Server ....") + go metrics.StartPrometheusMetricsServer(conf.CommonController.Metrics.Port) + } + loggers.LoggerAPKOperator.Info("Starting common controller ....") rateLimiterCache := xds.GetRateLimiterCache() diff --git a/common-controller/go.mod b/common-controller/go.mod index 80b975191..fa2172190 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -5,6 +5,8 @@ go 1.20 require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 + github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.0 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 @@ -19,7 +21,7 @@ require ( github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 - github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 + github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 google.golang.org/grpc v1.62.0 ) @@ -34,6 +36,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect @@ -41,13 +44,20 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vektah/gqlparser v1.3.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect @@ -84,10 +94,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 go.uber.org/multierr v1.11.0 // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index da9f79556..b98da4f27 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -66,6 +66,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -103,6 +105,7 @@ github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvR github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -138,6 +141,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= @@ -165,8 +170,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= @@ -179,6 +186,12 @@ github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -200,6 +213,10 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= @@ -210,6 +227,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -270,9 +289,11 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -281,6 +302,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/common-controller/internal/config/default_config.go b/common-controller/internal/config/default_config.go index 081122c36..40971fea6 100644 --- a/common-controller/internal/config/default_config.go +++ b/common-controller/internal/config/default_config.go @@ -41,6 +41,12 @@ var defaultConfig = &Config{ EventPort: 18000, RestPort: 18001, RetryInterval: 5, - Persistence: persistence{Type: "K8s"}}, + Persistence: persistence{Type: "K8s"}, + }, + Metrics: metrics{ + Enabled: false, + Type: "prometheus", + Port: 18006, + }, }, } diff --git a/common-controller/internal/config/types.go b/common-controller/internal/config/types.go index 372dc195e..35b57dd32 100644 --- a/common-controller/internal/config/types.go +++ b/common-controller/internal/config/types.go @@ -48,6 +48,7 @@ type commoncontroller struct { WebServer webServer InternalAPIServer internalAPIServer ControlPlane controlplane + Metrics metrics } type controlplane struct { Enabled bool @@ -102,3 +103,10 @@ type sts struct { type webServer struct { Port int64 } + +type metrics struct { + Enabled bool + Type string + Port int32 + CollectionInterval int32 +} diff --git a/common-controller/pkg/metrics/metrics.go b/common-controller/pkg/metrics/metrics.go new file mode 100644 index 000000000..c2da1fa33 --- /dev/null +++ b/common-controller/pkg/metrics/metrics.go @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/wso2/apk/adapter/pkg/logging" + logger "github.com/wso2/apk/common-controller/internal/loggers" + metrics "github.com/wso2/apk/common-go-libs/pkg/metrics" +) + +// StartPrometheusMetricsServer initializes and starts the metrics server to expose metrics to prometheus. +func StartPrometheusMetricsServer(port int32) { + + collector := metrics.CustomMetricsCollector() + prometheus.MustRegister(collector) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":"+strconv.Itoa(int(port)), nil) + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintln("Prometheus metrics server error:", err), + Severity: logging.MAJOR, + ErrorCode: 1110, + }) + } +} diff --git a/common-controller/pkg/metrics/types.go b/common-controller/pkg/metrics/types.go new file mode 100644 index 000000000..07761ebaf --- /dev/null +++ b/common-controller/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) diff --git a/common-go-libs/go.mod b/common-go-libs/go.mod index 37ab6563a..fcb0a6169 100644 --- a/common-go-libs/go.mod +++ b/common-go-libs/go.mod @@ -21,6 +21,16 @@ require ( sigs.k8s.io/gateway-api v0.7.1 ) +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect +) + require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -55,18 +65,19 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/common-go-libs/go.sum b/common-go-libs/go.sum index e4cdd32d2..b7ff16a46 100644 --- a/common-go-libs/go.sum +++ b/common-go-libs/go.sum @@ -43,6 +43,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -69,6 +71,7 @@ github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvR github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -95,6 +98,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -117,6 +122,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -131,6 +138,12 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -147,6 +160,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqfU= github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/wso2/apk/adapter v0.0.0-20231207051518-6dd728943082 h1:l+OdeDCNWPgie7L1fCjpfH04mAL3rFi4U+/idE8eduA= @@ -155,6 +172,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -206,17 +225,21 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= diff --git a/common-go-libs/pkg/metrics/metrics.go b/common-go-libs/pkg/metrics/metrics.go new file mode 100644 index 000000000..40abd594e --- /dev/null +++ b/common-go-libs/pkg/metrics/metrics.go @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package metrics holds the implementation for exposing adapter metrics to prometheus +package metrics + +import ( + "fmt" + "runtime" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/host" + "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v3/mem" + "github.com/wso2/apk/adapter/pkg/logging" + logger "github.com/wso2/apk/common-go-libs/loggers" +) + +var ( + prometheusMetricRegistry = prometheus.NewRegistry() +) + +// Collector contains the descriptions of the custom metrics exposed +type Collector struct { + internalRouteCount *prometheus.Desc + hostInfo *prometheus.Desc + availableCPUs *prometheus.Desc + freePhysicalMemory *prometheus.Desc + totalVirtualMemory *prometheus.Desc + usedVirtualMemory *prometheus.Desc + systemCPULoad *prometheus.Desc + loadAvg *prometheus.Desc +} + +// CustomMetricsCollector contains the descriptions of the custom metrics exposed +func CustomMetricsCollector() *Collector { + return &Collector{ + internalRouteCount: prometheus.NewDesc( + "internal_route_count", + "Number of internal routes created.", + nil, nil, + ), + hostInfo: prometheus.NewDesc( + "host_info", + "Host Info", + []string{"os"}, nil, + ), + availableCPUs: prometheus.NewDesc( + "os_available_cpu_total", + "Number of available CPUs.", + nil, nil, + ), + freePhysicalMemory: prometheus.NewDesc( + "os_free_physical_memory_bytes", + "Amount of free physical memory.", + nil, nil, + ), + totalVirtualMemory: prometheus.NewDesc( + "os_total_virtual_memory_bytes", + "Amount of total virtual memory.", + nil, nil, + ), + usedVirtualMemory: prometheus.NewDesc( + "os_used_virtual_memory_bytes", + "Amount of used virtual memory.", + nil, nil, + ), + systemCPULoad: prometheus.NewDesc( + "os_system_cpu_load_percentage", + "System-wide CPU usage as a percentage.", + nil, nil, + ), + loadAvg: prometheus.NewDesc( + "os_system_load_average", + "Current load of CPU in the host system for the last {x} minutes", + []string{"duration"}, nil, + ), + } +} + +// Describe sends all the descriptors of the metrics collected by this Collector +// to the provided channel. +func (collector *Collector) Describe(ch chan<- *prometheus.Desc) { + ch <- collector.internalRouteCount + ch <- collector.hostInfo + ch <- collector.availableCPUs + ch <- collector.freePhysicalMemory + ch <- collector.totalVirtualMemory + ch <- collector.usedVirtualMemory + ch <- collector.systemCPULoad + ch <- collector.loadAvg +} + +// Collect collects all the relevant Prometheus metrics. +func (collector *Collector) Collect(ch chan<- prometheus.Metric) { + + var hostInfoValue float64 + var availableCPUs float64 + var freePhysicalMemory float64 + var totalVirtualMemory float64 + var usedVirtualMemory float64 + var systemCPULoad float64 + + host, err := host.Info() + if handleError(err, "Failed to get host info") { + return + } + hostInfoValue = 1 + availableCPUs = float64(runtime.NumCPU()) + + v, err := mem.VirtualMemory() + if handleError(err, "Failed to read virtual memory metrics") { + return + } + freePhysicalMemory = float64(v.Free) + usedVirtualMemory = float64(v.Used) + totalVirtualMemory = float64(v.Total) + + percentages, err := cpu.Percent(0, false) + if handleError(err, "Failed to read cpu usage metrics") || len(percentages) == 0 { + return + } + totalPercentage := 0.0 + for _, p := range percentages { + totalPercentage += p + } + averagePercentage := totalPercentage / float64(len(percentages)) + systemCPULoad = averagePercentage + + avg, err := load.Avg() + if handleError(err, "Failed to read cpu load averages") { + return + } + + ch <- prometheus.MustNewConstMetric(collector.hostInfo, prometheus.GaugeValue, hostInfoValue, host.OS) + ch <- prometheus.MustNewConstMetric(collector.availableCPUs, prometheus.GaugeValue, availableCPUs) + ch <- prometheus.MustNewConstMetric(collector.freePhysicalMemory, prometheus.GaugeValue, freePhysicalMemory) + ch <- prometheus.MustNewConstMetric(collector.usedVirtualMemory, prometheus.GaugeValue, usedVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.totalVirtualMemory, prometheus.GaugeValue, totalVirtualMemory) + ch <- prometheus.MustNewConstMetric(collector.systemCPULoad, prometheus.GaugeValue, systemCPULoad) + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load1, "1m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load5, "5m") + ch <- prometheus.MustNewConstMetric(collector.loadAvg, prometheus.GaugeValue, avg.Load15, "15m") +} + +func init() { + // Register the Go collector with the registry + goCollector := collectors.NewGoCollector() + prometheusMetricRegistry.MustRegister(goCollector) +} + +func handleError(err error, message string) bool { + if err != nil { + logger.LoggerAPK.ErrorC(logging.ErrorDetails{ + Message: fmt.Sprintf(message, err.Error()), + Severity: logging.MINOR, + ErrorCode: 1109, + }) + return true + } + return false +} diff --git a/common-go-libs/pkg/metrics/types.go b/common-go-libs/pkg/metrics/types.go new file mode 100644 index 000000000..07761ebaf --- /dev/null +++ b/common-go-libs/pkg/metrics/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package metrics + +const ( + // PrometheusMetricType prometheus metric type + PrometheusMetricType = "prometheus" +) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle index 8d1b50f11..5a7bc193a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle +++ b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle @@ -71,6 +71,7 @@ dependencies { implementation libs.opentelemetry.extension.trace.propagators implementation libs.opentelemetry.sdk implementation libs.opentelemetry.semconv + implementation libs.prometheus implementation libs.snakeyaml implementation libs.sun.saaj.impl implementation libs.swagger.core.v3 diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java index 191a7bcee..5dc541a2c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/constants/MetadataConstants.java @@ -53,7 +53,7 @@ public class MetadataConstants { public static final String CLIENT_IP_KEY = WSO2_METADATA_PREFIX + "client-ip"; public static final String ERROR_CODE_KEY = "ErrorCode"; - public static final String CHOREO_CONNECT_ENFORCER_REPLY = "apk-enforcer-reply"; + public static final String APK_ENFORCER_REPLY = "apk-enforcer-reply"; public static final String RATELIMIT_WSO2_ORG_PREFIX = "customorg"; public static final String GATEWAY_URL = WSO2_METADATA_PREFIX + "x-original-gw-url"; public static final String API_ENVIRONMENT = WSO2_METADATA_PREFIX + "api-environment"; diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java index 9d751a267..e4311e4c9 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/discovery/JWTIssuerDiscoveryClient.java @@ -36,6 +36,8 @@ import org.wso2.apk.enforcer.discovery.service.subscription.JWTIssuerDiscoveryServiceGrpc; import org.wso2.apk.enforcer.discovery.subscription.JWTIssuer; import org.wso2.apk.enforcer.discovery.subscription.JWTIssuerList; +import org.wso2.apk.enforcer.jmx.JMXUtils; +import org.wso2.apk.enforcer.metrics.jmx.impl.ExtAuthMetrics; import org.wso2.apk.enforcer.subscription.SubscriptionDataHolder; import org.wso2.apk.enforcer.subscription.SubscriptionDataStore; import org.wso2.apk.enforcer.util.GRPCUtils; @@ -160,6 +162,9 @@ public void onNext(DiscoveryResponse response) { } subscriptionDataStore.addJWTIssuers(v); }); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordJWTIssuerMetrics(SubscriptionDataHolder.getInstance().getTotalJWTIssuerCount()); + } logger.info("Number of jwt issuers received : " + jwtIssuers.size()); ack(); } catch (Exception e) { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index f815f0203..f0f30364e 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -66,7 +66,7 @@ public class ExtAuthService extends AuthorizationGrpc.AuthorizationImplBase { public void check(CheckRequest request, StreamObserver responseObserver) { TracingSpan extAuthServiceSpan = null; Scope extAuthServiceSpanScope = null; - long starTimestamp = System.currentTimeMillis(); + long startTimestamp = System.currentTimeMillis(); try { String traceId = request.getAttributes().getRequest().getHttp() .getHeadersOrDefault(HttpConstants.X_REQUEST_ID_HEADER, @@ -94,10 +94,10 @@ public void check(CheckRequest request, StreamObserver responseOb } if (MetricsManager.isMetricsEnabled()) { MetricsExporter metricsExporter = MetricsManager.getInstance(); - metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - starTimestamp); + metricsExporter.trackMetric("enforcerLatency", System.currentTimeMillis() - startTimestamp); } if (JMXUtils.isJMXMetricsEnabled()) { - ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - starTimestamp); + ExtAuthMetrics.getInstance().recordMetric(System.currentTimeMillis() - startTimestamp); } } } @@ -142,7 +142,7 @@ private CheckResponse buildResponse(CheckRequest request, ResponseObject respons } addMetadata(metadataStructBuilder, "correlationID", responseObject.getCorrelationID()); - addMetadata(metadataStructBuilder, MetadataConstants.CHOREO_CONNECT_ENFORCER_REPLY, "Ok"); + addMetadata(metadataStructBuilder, MetadataConstants.APK_ENFORCER_REPLY, "Ok"); return checkResponseBuilder .setDynamicMetadata(metadataStructBuilder.build()) @@ -189,7 +189,7 @@ private CheckResponse buildResponse(CheckRequest request, ResponseObject respons responseObject.getRequestPath().split("\\?")[0]); // adding org level ratelimit key to metadata addMetadata(structBuilder, MetadataConstants.RATELIMIT_WSO2_ORG_PREFIX, responseObject.getOrganizationId()); - addMetadata(structBuilder, MetadataConstants.CHOREO_CONNECT_ENFORCER_REPLY, "Ok"); + addMetadata(structBuilder, MetadataConstants.APK_ENFORCER_REPLY, "Ok"); return CheckResponse.newBuilder().setStatus(Status.newBuilder().setCode(Code.OK_VALUE).build()) .setOkResponse(okResponseBuilder.build()) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java index 2c2028c8a..f460437ae 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/jmx/JMXUtils.java @@ -21,7 +21,7 @@ */ public class JMXUtils { - private static final String CHOREO_CONNECT_JMX_METRICS_ENABLE = "choreo.connect.jmx.metrics.enabled"; + private static final String APK_JMX_METRICS_ENABLE = "apk.jmx.metrics.enabled"; /** * Returns true if jmx metrics enabled as a system property, otherwise false. @@ -29,6 +29,6 @@ public class JMXUtils { * @return boolean */ public static boolean isJMXMetricsEnabled() { - return Boolean.getBoolean(CHOREO_CONNECT_JMX_METRICS_ENABLE); + return Boolean.getBoolean(APK_JMX_METRICS_ENABLE); } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java index 5bd3fc919..4755ebb45 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/api/ExtAuthMetricsMXBean.java @@ -23,40 +23,63 @@ public interface ExtAuthMetricsMXBean { /** * Getter for total request count. - * + * * @return long */ public long getTotalRequestCount(); /** * Getter for average response time in milli seconds. - * - * @return long + * + * @return double */ - public long getAverageResponseTimeMillis(); + public double getAverageResponseTimeMillis(); /** * Getter for maximum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMaxResponseTimeMillis(); + public double getMaxResponseTimeMillis(); /** * Getter for mimnimum response time in milliseconds. - * - * @return long + * + * @return double */ - public long getMinResponseTimeMillis(); + public double getMinResponseTimeMillis(); /** - * Resets all the metrics to thier initial values. + * Resets all the metrics to their initial values. */ public void resetExtAuthMetrics(); /** - * Resets all the metrics to thier initial values. + * Get request count in last five minutes window + * + * @return long + */ + public long getRequestCountInLastFiveMinuteWindow(); + + /** + * Get the start time of request count window + * + * @return + */ + public long getRequestCountWindowStartTimeMillis(); + + /** + * Get the number of token issuers + * + * @return + */ + public int getTokenIssuerCount(); + + /** + * Get the number of subscriptions + * + * @return */ - public long getRequestCountInLastFiveMinutes(); + public int getSubscriptionCount(); -} +} \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java index 9f558ba1f..be72384af 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/metrics/jmx/impl/ExtAuthMetrics.java @@ -29,34 +29,37 @@ public class ExtAuthMetrics extends TimerTask implements ExtAuthMetricsMXBean { private static final long REQUEST_COUNT_INTERVAL_MILLIS = 5 * 60 * 1000; private static ExtAuthMetrics extAuthMetricsMBean = null; - - private long requestCountInLastFiveMinutes = 0; + private long requestCountInLastFiveMinuteWindow = 0; + private int tokenIssuerCount = 0; + private long requestCountWindowStartTimeMillis = System.currentTimeMillis(); private long totalRequestCount = 0; - private long averageResponseTimeMillis = 0; - private long maxResponseTimeMillis = Long.MIN_VALUE; - private long minResponseTimeMillis = Long.MAX_VALUE; + private int subscriptionCount = 0; + private double averageResponseTimeMillis = 0; + private double maxResponseTimeMillis = Double.MIN_VALUE; + private double minResponseTimeMillis = Double.MAX_VALUE; private ExtAuthMetrics() { MBeanRegistrator.registerMBean(this); } - /** - * Getter for the Singleton ExtAuthMetrics instance. - * - * @return ExtAuthMetrics - */ - public static ExtAuthMetrics getInstance() { - if (extAuthMetricsMBean == null) { - synchronized (ExtAuthMetrics.class) { - if (extAuthMetricsMBean == null) { - Timer timer = new Timer(); - extAuthMetricsMBean = new ExtAuthMetrics(); - timer.schedule(extAuthMetricsMBean, 0, REQUEST_COUNT_INTERVAL_MILLIS); - } +/** + * Getter for the Singleton ExtAuthMetrics instance. + * + * @return ExtAuthMetrics + */ +public static ExtAuthMetrics getInstance() { + if (extAuthMetricsMBean == null) { + synchronized (ExtAuthMetrics.class) { + if (extAuthMetricsMBean == null) { + Timer timer = new Timer(); + extAuthMetricsMBean = new ExtAuthMetrics(); + extAuthMetricsMBean.requestCountWindowStartTimeMillis = System.currentTimeMillis(); + timer.schedule(extAuthMetricsMBean, REQUEST_COUNT_INTERVAL_MILLIS, REQUEST_COUNT_INTERVAL_MILLIS); } } - return extAuthMetricsMBean; } + return extAuthMetricsMBean; +} @Override public long getTotalRequestCount() { @@ -64,43 +67,68 @@ public long getTotalRequestCount() { }; @Override - public long getAverageResponseTimeMillis() { + public double getAverageResponseTimeMillis() { return averageResponseTimeMillis; }; @Override - public long getMaxResponseTimeMillis() { + public double getMaxResponseTimeMillis() { return maxResponseTimeMillis; }; @Override - public long getMinResponseTimeMillis() { + public double getMinResponseTimeMillis() { return minResponseTimeMillis; }; public synchronized void recordMetric(long responseTimeMillis) { - this.requestCountInLastFiveMinutes += 1; + this.requestCountInLastFiveMinuteWindow += 1; this.totalRequestCount += 1; - this.averageResponseTimeMillis = (this.averageResponseTimeMillis + responseTimeMillis) / totalRequestCount; + this.averageResponseTimeMillis = this.averageResponseTimeMillis + + (responseTimeMillis - this.averageResponseTimeMillis) / totalRequestCount; this.minResponseTimeMillis = Math.min(this.minResponseTimeMillis, responseTimeMillis); this.maxResponseTimeMillis = Math.max(this.maxResponseTimeMillis, responseTimeMillis); } + public synchronized void recordJWTIssuerMetrics(int jwtIssuers) { + this.tokenIssuerCount = jwtIssuers; + } + + public synchronized void recordSubscriptionMetrics(int subscriptionCount) { + this.subscriptionCount = subscriptionCount; + } + @Override public synchronized void resetExtAuthMetrics() { this.totalRequestCount = 0; this.averageResponseTimeMillis = 0; - this.maxResponseTimeMillis = Long.MIN_VALUE; - this.minResponseTimeMillis = Long.MAX_VALUE; + this.maxResponseTimeMillis = Double.MIN_VALUE; + this.minResponseTimeMillis = Double.MAX_VALUE; } @Override public synchronized void run() { - requestCountInLastFiveMinutes = 0; + requestCountWindowStartTimeMillis = System.currentTimeMillis(); + requestCountInLastFiveMinuteWindow = 0; } @Override - public long getRequestCountInLastFiveMinutes() { - return requestCountInLastFiveMinutes; + public long getRequestCountInLastFiveMinuteWindow() { + return requestCountInLastFiveMinuteWindow; } -} + + @Override + public long getRequestCountWindowStartTimeMillis() { + return requestCountWindowStartTimeMillis; + } + + @Override + public int getTokenIssuerCount() { + return tokenIssuerCount; + } + + @Override + public int getSubscriptionCount() { + return subscriptionCount; + } +} \ No newline at end of file diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java index 4454ef588..c90bdcf14 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataHolder.java @@ -27,7 +27,8 @@ public class SubscriptionDataHolder { private static final SubscriptionDataHolder instance = new SubscriptionDataHolder(); - Map subscriptionDataStoreMap = new HashMap<>(); + Map subscriptionDataStoreMap = new HashMap<>(); + public static SubscriptionDataHolder getInstance() { return instance; } @@ -35,9 +36,27 @@ public static SubscriptionDataHolder getInstance() { public SubscriptionDataStore getSubscriptionDataStore(String organization) { return subscriptionDataStoreMap.get(organization); } + public SubscriptionDataStore initializeSubscriptionDataStore(String organization) { SubscriptionDataStore subscriptionDataStore = new SubscriptionDataStoreImpl(); - subscriptionDataStoreMap.put(organization,subscriptionDataStore); + subscriptionDataStoreMap.put(organization, subscriptionDataStore); return subscriptionDataStore; } + + public int getTotalSubscriptionCount() { + int totalSubCount = 0; + for (SubscriptionDataStore store : subscriptionDataStoreMap.values()) { + totalSubCount += store.getSubscriptionCount(); + } + return totalSubCount; + } + + public int getTotalJWTIssuerCount() { + int totalJWTIssuerCount = 0; + for (SubscriptionDataStore store : subscriptionDataStoreMap.values()) { + totalJWTIssuerCount += store.getJWTIssuerCount(); + } + return totalJWTIssuerCount; + } + } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java index 33be66ea2..3fca55d2d 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStore.java @@ -51,7 +51,6 @@ public interface SubscriptionDataStore { void addApplications(List applicationList); - void addApplicationKeyMappings( List applicationKeyMappingList); @@ -73,7 +72,7 @@ void addApplicationKeyMappings( * @return ApplicationKeyMapping which match the given parameters */ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentifier, String keyType, - String securityScheme, String envType); + String securityScheme, String envType); /** * Filter the applications map based on the provided parameters. @@ -108,11 +107,13 @@ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentif void addApplicationMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping applicationMapping); - void addApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); + void addApplicationKeyMapping( + org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); void removeApplicationMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping applicationMapping); - void removeApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); + void removeApplicationKeyMapping( + org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping applicationKeyMapping); void removeSubscription(org.wso2.apk.enforcer.discovery.subscription.Subscription subscription); @@ -120,4 +121,7 @@ ApplicationKeyMapping getMatchingApplicationKeyMapping(String applicationIdentif public void addApplicationMappings(List applicationMappingList); + int getSubscriptionCount(); + + int getJWTIssuerCount(); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java index 33c132e65..04eaf413c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java @@ -370,6 +370,7 @@ public void removeApplicationKeyMapping(org.wso2.apk.enforcer.discovery.subscrip public void removeSubscription(org.wso2.apk.enforcer.discovery.subscription.Subscription subscription) { subscriptionMap.remove(subscription.getUuid()); + } @Override @@ -398,4 +399,16 @@ private String getMapKey(String environment, String issuer) { return environment + DELEM_PERIOD + issuer; } + @Override + public int getSubscriptionCount() { + + return subscriptionMap.size(); + } + + @Override + public int getJWTIssuerCount() { + + return jwtValidatorMap.size(); + } + } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java index 22a576d07..de2aa497d 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreUtil.java @@ -29,6 +29,8 @@ import org.wso2.apk.enforcer.discovery.subscription.ApplicationKeyMapping; import org.wso2.apk.enforcer.discovery.subscription.ApplicationMapping; import org.wso2.apk.enforcer.discovery.subscription.Subscription; +import org.wso2.apk.enforcer.jmx.JMXUtils; +import org.wso2.apk.enforcer.metrics.jmx.impl.ExtAuthMetrics; import org.wso2.apk.enforcer.util.ApacheFeignHttpClient; import org.wso2.apk.enforcer.util.FilterUtils; @@ -157,6 +159,9 @@ private static void loadSubscriptions() { new Thread(() -> { SubscriptionListDto subscriptions = subscriptionValidationDataRetrievalRestClient.getAllSubscriptions(); List list = subscriptions.getList(); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); + } Map> orgWizeMAp = new HashMap<>(); for (SubscriptionDto subscriptionDto : list) { String organization = subscriptionDto.getOrganization(); @@ -188,6 +193,9 @@ public static void addSubscription(Subscription subscription) { SubscriptionDataStore subscriptionDataStore = getSubscriptionDataStore(subscription.getOrganization()); subscriptionDataStore.addSubscription(subscription); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); + } } @@ -240,8 +248,10 @@ public static void removeApplicationKeyMapping(ApplicationKeyMapping application public static void removeSubscription(Subscription subscription) { SubscriptionDataStore subscriptionDataStore = getSubscriptionDataStore(subscription.getOrganization()); - subscriptionDataStore.removeSubscription(subscription); + if (JMXUtils.isJMXMetricsEnabled()) { + ExtAuthMetrics.getInstance().recordSubscriptionMetrics(SubscriptionDataHolder.getInstance().getTotalSubscriptionCount()); + } } diff --git a/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml new file mode 100644 index 000000000..5771c0027 --- /dev/null +++ b/gateway/enforcer/resources/metrics/prometheus-jmx-config.yml @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------- +# Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + +# Config options: https://github.com/prometheus/jmx_exporter/blob/main/README.md +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + # WSO2 APK related metrics + - pattern: "org.wso2.apk.enforcer<>total_request_count: (.*)" + name: org_wso2_apk_enforcer_request_count_total + help: "WSO2 APK enforcer total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: "org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis|token_issuer_count|subscription_count): (.*)" + name: org_wso2_apk_enforcer_$1 + help: "WSO2 APK enforcer $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' + name: org_wso2_apk_enforcer_thread_pool_$1 + help: "WSO2 APK enforcer thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 89a53a89d..6a4e56650 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -25,7 +25,19 @@ data: [ballerina.log] level = "DEBUG" [ballerina.http] - traceLogConsole = true + traceLogConsole = false + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [ballerina.observe] + metricsEnabled=true + metricsReporter="prometheus" + [ballerinax.prometheus] + port=18006 + {{if .Values.wso2.apk.metrics.configDSBalHost}} + host="{{ .Values.wso2.apk.metrics.configDSBalHost}}" + {{else}} + host="0.0.0.0" + {{end}} + {{end}} [wso2.config_deployer_service.keyStores.tls] keyFilePath = "/home/wso2apk/config-deployer/security/config.key" certFilePath = "/home/wso2apk/config-deployer/security/config.pem" diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml index 29d23a6df..27d71680b 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-deployment.yaml @@ -42,6 +42,12 @@ spec: ports: - containerPort: 9443 protocol: "TCP" + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18006 + protocol: "TCP" + - containerPort: 18007 + protocol: "TCP" + {{ end }} readinessProbe: httpGet: path: /health @@ -61,6 +67,9 @@ spec: - mountPath: /home/wso2apk/config-deployer/conf/Config.toml name: config-toml-volume subPath: Config.toml + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-configds.yml + subPath: prometheus-jmx-config-configds.yml - name: config-ds-tls-volume mountPath: /home/wso2apk/config-deployer/security/config.key {{- if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} @@ -105,6 +114,9 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-config-ds-configmap + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-configds - name: config-ds-tls-volume secret: {{ if and .Values.wso2.apk.dp.configdeployer.configs .Values.wso2.apk.dp.configdeployer.configs.tls }} diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..de2731309 --- /dev/null +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-prometheus-jmx-configmap.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-configds +data: + prometheus-jmx-config-configds.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml index 1f28bf973..7bf0a25fc 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/adapter-deployment.yaml @@ -46,6 +46,10 @@ spec: protocol: "TCP" - containerPort: 18001 protocol: TCP + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + - containerPort: 18006 + protocol: "TCP" + {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.adapter.deployment.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.adapter.deployment.env | indent 10 }} - name: OPERATOR_POD_NAMESPACE diff --git a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml index 5a2647ca0..eead36fcc 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-controller/common-controller-deployment.yaml @@ -48,6 +48,10 @@ spec: protocol: TCP - containerPort: 18003 protocol: TCP + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18006 + protocol: "TCP" + {{ end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.commonController.deployment.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.commonController.deployment.env | indent 10 }} - name: OPERATOR_POD_NAMESPACE diff --git a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml index 0fa0bb360..12777a55d 100644 --- a/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/common-log-conf.yaml @@ -10,6 +10,12 @@ data: {{- if and .Values.wso2.apk.dp.environment .Values.wso2.apk.dp.environment.name }} environment = "{{ .Values.wso2.apk.dp.environment.name }}" {{- end }} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [commoncontroller.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18006 + {{ end}} [commoncontroller.server] label = "ratelimiter" {{ if and .Values.wso2.apk.dp.commonController.configs .Values.wso2.apk.dp.commonController.configs.apiNamespaces }} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index a9918a9a5..bf2883c1e 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -52,6 +52,10 @@ spec: protocol: "TCP" - containerPort: 18002 protocol: "TCP" + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + - containerPort: 18006 + protocol: "TCP" + {{- end }} {{ include "apk-helm.deployment.resources" .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.resources | indent 10 }} {{ include "apk-helm.deployment.env" .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.env | indent 10 }} - name: ADAPTER_HOST_NAME @@ -87,7 +91,11 @@ spec: - name: enforcer_admin_pwd value: admin - name: JAVA_OPTS + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml + {{- else }} value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + {{- end }} {{- if .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.redis }} - name: REDIS_USERNAME value: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.redis.username | default "default" }} @@ -153,6 +161,11 @@ spec: {{- else }} subPath: tls.key {{- end }} + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-enforcer.yml + subPath: prometheus-jmx-config-enforcer.yml + {{- end }} - name: enforcer-keystore-secret-volume mountPath: /home/wso2/security/keystore/enforcer.crt {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} @@ -382,9 +395,14 @@ spec: # secretName: {{ template "apk-helm.resource.prefix" . }}-common-controller-server-cert # {{- end }} # defaultMode: 420 + {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-enforcer + {{- end }} - name: enforcer-keystore-secret-volume secret: - {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} + {{- if and .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls }} secretName: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.enforcer.configs.tls.certificatesSecret | default (printf "%s-enforcer-server-cert" (include "apk-helm.resource.prefix" .)) }} {{- else }} secretName: {{ template "apk-helm.resource.prefix" . }}-enforcer-server-cert diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..4f45e2d4f --- /dev/null +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/prometheus-jmx-configmap.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-enforcer +data: + prometheus-jmx-config-enforcer.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # WSO2 APK related metrics + - pattern: 'org.wso2.apk.enforcer<>total_request_count: (.*)' + name: org_wso2_apk_enforcer_request_count_total + help: "WSO2 APK enforcer total request count." + attrNameSnakeCase: true + type: COUNTER + - pattern: 'org.wso2.apk.enforcer<>(average_response_time_millis|max_response_time_millis|min_response_time_millis|request_count_in_last_five_minute_window|request_count_window_start_time_millis|token_issuer_count|subscription_count): (.*)' + name: org_wso2_apk_enforcer_$1 + help: "WSO2 APK enforcer $1." + attrNameSnakeCase: true + type: GAUGE + - pattern: 'org.wso2.apk.enforcer<>(\w+): (.*)' + name: org_wso2_apk_enforcer_thread_pool_$1 + help: "WSO2 APK enforcer thread pool $1." + attrNameSnakeCase: true + type: GAUGE + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE \ No newline at end of file diff --git a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml index 7b775168b..2fc6b24fd 100644 --- a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml @@ -14,6 +14,12 @@ data: [adapter.operator] namespaces = [{{ include "commaJoinedQuotedList" .Values.wso2.apk.dp.adapter.configs.apiNamespaces}}] {{ end}} + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [adapter.metrics] + enabled = {{.Values.wso2.apk.metrics.enabled}} + type = "{{.Values.wso2.apk.metrics.type| default "prometheus" }}" + port = 18006 + {{ end}} {{ if and .Values.wso2.apk.dp.gatewayRuntime.deployment .Values.wso2.apk.dp.gatewayRuntime.deployment.router .Values.wso2.apk.dp.gatewayRuntime.deployment.router.configs }} [router] diff --git a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml index 7f15be10f..a56500cd5 100644 --- a/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml +++ b/helm-charts/templates/data-plane/ratelimiter/ratelimiter-deployment.yaml @@ -35,6 +35,14 @@ spec: automountServiceAccountToken: false serviceAccountName: {{ .Values.wso2.apk.auth.serviceAccountName }} containers: + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd }} + - name: statsd + image: {{ .Values.wso2.apk.metrics.statsd.image.repository }}:{{ .Values.wso2.apk.metrics.statsd.image.tag }} + imagePullPolicy: {{ .Values.wso2.apk.metrics.statsd.imagePullPolicy }} + ports: + - name: statsd-metrics + containerPort: 9102 + {{ end }} - name: ratelimiter image: {{ .Values.wso2.apk.dp.ratelimiter.deployment.image }} imagePullPolicy: {{ .Values.wso2.apk.dp.ratelimiter.deployment.imagePullPolicy }} @@ -51,8 +59,17 @@ spec: value: "0.0.0.0" - name: PORT value: "8090" + {{if and .Values.wso2.apk.metrics.enabled .Values.wso2.apk.metrics.statsd}} + - name: USE_STATSD + value: "true" + - name: STATSD_HOST + value: "0.0.0.0" + - name: STATSD_PORT + value: "9125" + {{ else }} - name: USE_STATSD value: "false" + {{ end }} - name: LOG_LEVEL value: "DEBUG" - name: LIMIT_RESPONSE_HEADERS_ENABLED diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml index 2abf4dcf8..62c690912 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml @@ -25,7 +25,19 @@ data: [ballerina.log] level = "DEBUG" [ballerina.http] - traceLogConsole = true + traceLogConsole = false + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [ballerina.observe] + metricsEnabled=true + metricsReporter="prometheus" + [ballerinax.prometheus] + port=18006 + {{if .Values.wso2.apk.metrics.idpDSBalHost}} + host="{{ .Values.wso2.apk.metrics.idpDSBalHost}}" + {{else}} + host="0.0.0.0" + {{end}} + {{end}} # Sample configurations [wso2.idp_domain_service.datasourceConfiguration] description = "Database for idp" diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml index d246dd99d..466b4da4a 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml @@ -51,6 +51,12 @@ spec: ports: - containerPort: 9443 protocol: "TCP" + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18006 + protocol: "TCP" + - containerPort: 18007 + protocol: "TCP" + {{ end }} readinessProbe: httpGet: path: /health @@ -76,6 +82,9 @@ spec: - mountPath: /home/wso2apk/idp/security/wso2carbon.pem name: idp-signing-keystore-volume subPath: wso2carbon.pem + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-idpds.yml + subPath: prometheus-jmx-config-idpds.yml - name: idp-ds-tls-volume mountPath: /home/wso2apk/idp/security/idp.key {{- if and .Values.idp.idpds.configs .Values.idp.idpds.configs.tls }} @@ -111,6 +120,9 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-idp-ds-configmap + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-idpds - name: idp-signing-keystore-volume secret: secretName: {{ template "apk-helm.resource.prefix" . }}-idp-signing-keystore-secret diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml new file mode 100644 index 000000000..773615b87 --- /dev/null +++ b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-idpds +data: + prometheus-jmx-config-idpds.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index dadb28be3..bbc801126 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -249,6 +249,23 @@ wso2: # JWKSClient: # skipSSLVerification: false # hostnameVerifier: "AllowAll" + + metrics: + enabled: false + # configDSBalHost: 0.0.0.0 + # idpDSBalHost: 0.0.0.0 + # statsd: + # image: + # repository: prom/statsd-exporter + # tag: v0.26.0 + # imagePullPolicy: IfNotPresent + # resources: + # limits: + # memory: 128Mi + # requests: + # cpu: 0.1 + # memory: 64Mi + idp: enabled: true listener: diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 0c04fc44c..550602951 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -592,6 +592,26 @@ wso2: keepAliveTime: 600 # -- Queue size of the worker threads queueSize: 1000 + metrics: + # -- Enable Prometheus metrics + enabled: false + # -- Configure the host for exposing the config ds ballerina metrics + # configDSBalHost: 0.0.0.0 + # -- Configure the host for exposing the idp ds ballerina metrics. + # idpDSBalHost: 0.0.0.0 + # -- Statsd is required to expose metrics from ratelimiter + # statsd: + # image: + # repository: prom/statsd-exporter + # tag: v0.26.0 + # imagePullPolicy: IfNotPresent + # resources: + # limits: + # memory: 128Mi + # requests: + # cpu: 0.1 + # memory: 64Mi + idp: # -- Enable Non production identity server enabled: true diff --git a/idp/idp-domain-service/ballerina/Ballerina.toml b/idp/idp-domain-service/ballerina/Ballerina.toml index 7f9e44b8e..ac2a6ec1e 100644 --- a/idp/idp-domain-service/ballerina/Ballerina.toml +++ b/idp/idp-domain-service/ballerina/Ballerina.toml @@ -9,5 +9,10 @@ groupId = "org.postgresql" artifactId = "postgresql" version = "42.2.20" +[[platform.java11.dependency]] +groupId = "io.prometheus.jmx" +artifactId = "jmx_prometheus_javaagent" +version = "0.20.0" + [build-options] observabilityIncluded = true diff --git a/idp/idp-domain-service/ballerina/Dependencies.toml b/idp/idp-domain-service/ballerina/Dependencies.toml index fc72aa1d3..210772f7e 100644 --- a/idp/idp-domain-service/ballerina/Dependencies.toml +++ b/idp/idp-domain-service/ballerina/Dependencies.toml @@ -375,6 +375,21 @@ modules = [ {org = "ballerinax", packageName = "postgresql", moduleName = "postgresql"} ] +[[package]] +org = "ballerinax" +name = "prometheus" +version = "0.2.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "regex"} +] +modules = [ + {org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"} +] + [[package]] org = "wso2" name = "idp_domain_service" @@ -392,7 +407,8 @@ dependencies = [ {org = "ballerina", name = "url"}, {org = "ballerina", name = "uuid"}, {org = "ballerinai", name = "observe"}, - {org = "ballerinax", name = "postgresql"} + {org = "ballerinax", name = "postgresql"}, + {org = "ballerinax", name = "prometheus"} ] modules = [ {org = "wso2", packageName = "idp_domain_service", moduleName = "idp_domain_service"} diff --git a/idp/idp-domain-service/ballerina/Init.bal b/idp/idp-domain-service/ballerina/Init.bal index b34ebe42a..78fe6721e 100644 --- a/idp/idp-domain-service/ballerina/Init.bal +++ b/idp/idp-domain-service/ballerina/Init.bal @@ -20,6 +20,7 @@ import ballerina/log; import ballerina/sql; import ballerina/http; import ballerina/jwt; +import ballerinax/prometheus as _; configurable IDPConfiguration & readonly idpConfiguration = ?; final postgresql:Client|sql:Error dbClient; diff --git a/idp/idp-domain-service/build.gradle b/idp/idp-domain-service/build.gradle index 612190f25..cfe38cb8e 100644 --- a/idp/idp-domain-service/build.gradle +++ b/idp/idp-domain-service/build.gradle @@ -36,7 +36,7 @@ release { unSnapshotVersion.finalizedBy ":ballerina:commit_toml_files" afterReleaseBuild.dependsOn ":docker:docker_push" task build{ - dependsOn("docker:build") + dependsOn("docker:buildDockerImage") } build.configure { dependsOn(":docker:build") diff --git a/idp/idp-domain-service/docker/Dockerfile b/idp/idp-domain-service/docker/Dockerfile index 6f8406c82..f07cf7498 100644 --- a/idp/idp-domain-service/docker/Dockerfile +++ b/idp/idp-domain-service/docker/Dockerfile @@ -72,6 +72,7 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \ ADD idp ${USER_HOME}/idp COPY docker-entrypoint.sh ${USER_HOME} +COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/ RUN chown -R ${USER} ${USER_HOME}/idp \ && chown ${USER} /home/${USER}/docker-entrypoint.sh \ diff --git a/idp/idp-domain-service/docker/build.gradle b/idp/idp-domain-service/docker/build.gradle index 75f812f75..25d8b11f0 100644 --- a/idp/idp-domain-service/docker/build.gradle +++ b/idp/idp-domain-service/docker/build.gradle @@ -18,20 +18,45 @@ apply from: "$projectDir/../../../common-gradle-scripts/docker.gradle" apply from: "$rootDir/../../common-gradle-scripts/copy.gradle" +apply from: "$projectDir/../../../common-gradle-scripts/java.gradle" +dependencies { + implementation libs.prometheus +} + +tasks.register('copy_dependencies', Copy) { + dependsOn create_lib + from configurations.default + into "$projectDir/../docker/lib" + finalizedBy build +} + +tasks.register('create_lib') { + dependsOn delete_lib + finalizedBy copy_dependencies +} + +tasks.register('delete_lib', Delete) { + delete "$projectDir/lib" + finalizedBy create_lib +} + +tasks.named('build').configure { + dependsOn copy_dependencies +} tasks.named('copy_dist').configure{ finalizedBy docker_build } -task build{ +task buildDockerImage{ group 'build' description 'Build docker image' dependsOn 'copy_dist' dependsOn 'docker_build' } -build.configure{ +buildDockerImage.configure{ mustRunAfter(":ballerina:build") dependsOn(":ballerina:build") - } \ No newline at end of file +} \ No newline at end of file diff --git a/idp/idp-domain-service/docker/idp/idp.sh b/idp/idp-domain-service/docker/idp/idp.sh index ffa63a2c1..dc8c6e8bc 100755 --- a/idp/idp-domain-service/docker/idp/idp.sh +++ b/idp/idp-domain-service/docker/idp/idp.sh @@ -98,9 +98,10 @@ fi echo "Using Java memory options: $JVM_MEM_OPTS" $JAVACMD \ - $JVM_MEM_OPTS \ - $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Djava.io.tmpdir="$IDP_HOME/tmp" \ - -jar idp_domain_service.jar $* - status=$? \ No newline at end of file + $JVM_MEM_OPTS \ + $JAVA_OPTS \ + -classpath "$CLASSPATH" \ + -Djava.io.tmpdir="$IDP_HOME/tmp" \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-idpds.yml \ + -jar idp_domain_service.jar $* +status=$? diff --git a/libs.versions.toml b/libs.versions.toml index 98a8315ee..100dd115a 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -80,6 +80,7 @@ opentelemetry-extension-trace-propagators = {module = "io.opentelemetry:opentele opentelemetry-sdk = {module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry"} opentelemetry-semconv = {module = "io.opentelemetry:opentelemetry-semconv", version.ref = "opentelemetry-semconv"} postgresql = {module = "org.postgresql:postgresql", version.ref = "postgresql"} +prometheus = {module = "io.prometheus.jmx:jmx_prometheus_javaagent", version.ref = "prometheus"} snakeyaml = {module = "org.yaml:snakeyaml", version.ref = "snakeyaml"} sun-saaj-impl = {module = "com.sun.xml.messaging.saaj:saaj-impl", version.ref = "sun"} swagger-annotations = {module = "io.swagger:swagger-annotations", version.ref = "io-swagger"} @@ -165,6 +166,7 @@ opentelemetry-jaeger-thrift = "1.24.0" opentelemetry-semconv = "1.24.0-alpha" org-json = "20231013" postgresql = "42.5.0" +prometheus = "0.20.0" snakeyaml = "2.0" sun = "1.5.3" swagger-codegen = "2.4.11" diff --git a/runtime/config-deployer-service/ballerina/APIClient.bal b/runtime/config-deployer-service/ballerina/APIClient.bal index d6441725f..c0009d947 100644 --- a/runtime/config-deployer-service/ballerina/APIClient.bal +++ b/runtime/config-deployer-service/ballerina/APIClient.bal @@ -27,10 +27,9 @@ import ballerina/uuid; import ballerina/crypto; import config_deployer_service.java.io as javaio; import wso2/apk_common_lib as commons; - -# +import ballerinax/prometheus as _; public class APIClient { - + # This function used to convert APKInternalAPI model to APKConf. # # + api - APKInternalAPI model diff --git a/runtime/config-deployer-service/ballerina/Ballerina.toml b/runtime/config-deployer-service/ballerina/Ballerina.toml index 46bc216ad..82f0b4f11 100644 --- a/runtime/config-deployer-service/ballerina/Ballerina.toml +++ b/runtime/config-deployer-service/ballerina/Ballerina.toml @@ -5,7 +5,7 @@ version = "1.1.0-SNAPSHOT" distribution = "2201.8.0" [build-options] -observabilityIncluded = false +observabilityIncluded = true [[dependency]] org = "wso2" @@ -49,6 +49,12 @@ groupId = "io.swagger" artifactId = "swagger-core" version = "1.6.10" +# transitive dependency of io.prometheus.jmx:jmx_prometheus_javaagent:0.20.0 +[[platform.java11.dependency]] +groupId = "io.prometheus.jmx" +artifactId = "jmx_prometheus_javaagent" +version = "0.20.0" + # transitive dependency of io.swagger.parser.v3:swagger-parser:2.1.13 [[platform.java11.dependency]] groupId = "io.swagger" diff --git a/runtime/config-deployer-service/ballerina/Dependencies.toml b/runtime/config-deployer-service/ballerina/Dependencies.toml index 9c45a8850..bb1ba5577 100644 --- a/runtime/config-deployer-service/ballerina/Dependencies.toml +++ b/runtime/config-deployer-service/ballerina/Dependencies.toml @@ -365,6 +365,18 @@ modules = [ {org = "ballerina", packageName = "uuid", moduleName = "uuid"} ] +[[package]] +org = "ballerinai" +name = "observe" +version = "0.0.0" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "observe"} +] +modules = [ + {org = "ballerinai", packageName = "observe", moduleName = "observe"} +] + [[package]] org = "ballerinax" name = "postgresql" @@ -377,6 +389,21 @@ dependencies = [ {org = "ballerina", name = "time"} ] +[[package]] +org = "ballerinax" +name = "prometheus" +version = "0.2.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "regex"} +] +modules = [ + {org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"} +] + [[package]] org = "wso2" name = "apk_common_lib" @@ -420,6 +447,8 @@ dependencies = [ {org = "ballerina", name = "test"}, {org = "ballerina", name = "url"}, {org = "ballerina", name = "uuid"}, + {org = "ballerinai", name = "observe"}, + {org = "ballerinax", name = "prometheus"}, {org = "wso2", name = "apk_common_lib"} ] modules = [ @@ -432,6 +461,7 @@ modules = [ {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.model"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.api"}, + {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.definitions"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.org.wso2.apk.config.model"}, {org = "wso2", packageName = "config_deployer_service", moduleName = "config_deployer_service.partitionClient"} ] diff --git a/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal b/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal index 437a369e8..11907a88d 100644 --- a/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal +++ b/runtime/config-deployer-service/ballerina/artifact-deployer-api_service.bal @@ -1,4 +1,6 @@ import ballerina/http; +import ballerinax/prometheus as _; + import wso2/apk_common_lib as commons; isolated service http:InterceptableService /api/deployer on ep0 { @@ -9,11 +11,11 @@ isolated service http:InterceptableService /api/deployer on ep0 { # anydata (API deployed successfully) # BadRequestError (Bad Request. Invalid request or validation error.) # InternalServerErrorError (Internal Server Error.) - isolated resource function post apis/deploy(http:RequestContext requestContext,http:Request request) returns commons:APKError|http:Response { + isolated resource function post apis/deploy(http:RequestContext requestContext, http:Request request) returns commons:APKError|http:Response { DeployerClient deployerClient = new; commons:UserContext authenticatedUserContext = check commons:getAuthenticatedUserContext(requestContext); commons:Organization organization = authenticatedUserContext.organization; - return check deployerClient.handleAPIDeployment(request,organization); + return check deployerClient.handleAPIDeployment(request, organization); } # Undeploy API # @@ -22,11 +24,11 @@ isolated service http:InterceptableService /api/deployer on ep0 { # AcceptedString (API undeployed successfully) # BadRequestError (Bad Request. Invalid request or validation error.) # InternalServerErrorError (Internal Server Error.) - isolated resource function post apis/undeploy(http:RequestContext requestContext,string apiId) returns AcceptedString|BadRequestError|InternalServerErrorError|commons:APKError { + isolated resource function post apis/undeploy(http:RequestContext requestContext, string apiId) returns AcceptedString|BadRequestError|InternalServerErrorError|commons:APKError { DeployerClient deployerClient = new; commons:UserContext authenticatedUserContext = check commons:getAuthenticatedUserContext(requestContext); commons:Organization organization = authenticatedUserContext.organization; - return check deployerClient.handleAPIUndeployment(apiId,organization); + return check deployerClient.handleAPIUndeployment(apiId, organization); } public function createInterceptors() returns http:Interceptor|http:Interceptor[] { diff --git a/runtime/config-deployer-service/docker/Dockerfile b/runtime/config-deployer-service/docker/Dockerfile index 82f8ba2aa..a7df67708 100644 --- a/runtime/config-deployer-service/docker/Dockerfile +++ b/runtime/config-deployer-service/docker/Dockerfile @@ -53,6 +53,8 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \ ADD config-deployer ${USER_HOME}/config-deployer COPY docker-entrypoint.sh ${USER_HOME} +COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/ + RUN chown -R ${USER} ${USER_HOME}/config-deployer \ && chown ${USER} /home/${USER}/docker-entrypoint.sh \ && chgrp -R 0 ${USER_HOME} \ diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index cfb35a976..3d9253f7e 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -2,23 +2,26 @@ PRG="$0" while [ -h "$PRG" ]; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then + ls=$(ls -ld "$PRG") + link=$(expr "$ls" : '.*-> \(.*\)$') + if expr "$link" : '.*/.*' >/dev/null; then PRG="$link" else - PRG=`dirname "$PRG"`/"$link" + PRG=$(dirname "$PRG")/"$link" fi done # Get standard environment variables -PRGDIR=`dirname "$PRG"` +PRGDIR=$(dirname "$PRG") -[ -z "$RUNTIME_HOME" ] && RUNTIME_HOME=`cd "$PRGDIR" ; pwd` +[ -z "$RUNTIME_HOME" ] && RUNTIME_HOME=$( + cd "$PRGDIR" + pwd +) -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then +if [ -z "$JAVACMD" ]; then + if [ -n "$JAVA_HOME" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else @@ -29,7 +32,7 @@ if [ -z "$JAVACMD" ] ; then fi fi -if [ ! -x "$JAVACMD" ] ; then +if [ ! -x "$JAVACMD" ]; then echo "Error: JAVA_HOME is not defined correctly." echo " Admin cannot execute $JAVACMD" exit 1 @@ -42,16 +45,15 @@ if [ -z "$JAVA_HOME" ]; then fi # ----- Process the input command ---------------------------------------------- args="" -for c in $* -do - if [ "$c" = "--debug" ] || [ "$c" = "-debug" ] || [ "$c" = "debug" ]; then - CMD="--debug" - continue - elif [ "$CMD" = "--debug" ]; then - if [ -z "$PORT" ]; then - PORT=$c - fi +for c in $*; do + if [ "$c" = "--debug" ] || [ "$c" = "-debug" ] || [ "$c" = "debug" ]; then + CMD="--debug" + continue + elif [ "$CMD" = "--debug" ]; then + if [ -z "$PORT" ]; then + PORT=$c fi + fi done if [ "$CMD" = "--debug" ]; then @@ -69,11 +71,10 @@ fi CLASSPATH="" if [ -e "$JAVA_HOME/lib/tools.jar" ]; then - CLASSPATH="$JAVA_HOME/lib/tools.jar" + CLASSPATH="$JAVA_HOME/lib/tools.jar" fi -for t in "$RUNTIME_HOME"/lib/*.jar -do - CLASSPATH="$CLASSPATH":$t +for t in "$RUNTIME_HOME"/lib/*.jar; do + CLASSPATH="$CLASSPATH":$t done # ----- Execute The Requested Command ----------------------------------------- @@ -85,22 +86,23 @@ cd "$RUNTIME_HOME" TMP_DIR="$RUNTIME_HOME"/tmp if [ -d "$TMP_DIR" ]; then -rm -rf "$TMP_DIR"/* + rm -rf "$TMP_DIR"/* fi START_EXIT_STATUS=121 status=$START_EXIT_STATUS if [ -z "$JVM_MEM_OPTS" ]; then - java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') - JVM_MEM_OPTS="-Xms256m -Xmx1024m" + java_version=$("$JAVACMD" -version 2>&1 | awk -F '"' '/version/ {print $2}') + JVM_MEM_OPTS="-Xms256m -Xmx1024m" fi echo "Using Java memory options: $JVM_MEM_OPTS" $JAVACMD \ - $JVM_MEM_OPTS \ - $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ - -jar config_deployer_service.jar $* - status=$? \ No newline at end of file + $JVM_MEM_OPTS \ + $JAVA_OPTS \ + -classpath "$CLASSPATH" \ + -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18007:/tmp/metrics/prometheus-jmx-config-configds.yml \ + -jar config_deployer_service.jar $* +status=$? diff --git a/runtime/config-deployer-service/java/build.gradle b/runtime/config-deployer-service/java/build.gradle index 95e278c66..a7f868785 100644 --- a/runtime/config-deployer-service/java/build.gradle +++ b/runtime/config-deployer-service/java/build.gradle @@ -17,8 +17,6 @@ apply from: "$projectDir/../../../common-gradle-scripts/java.gradle" - - dependencies { implementation libs.commons.logging implementation libs.gson @@ -34,4 +32,35 @@ dependencies { implementation libs.handy.uri.templates implementation libs.json implementation libs.commons.lang + implementation libs.prometheus +} + +tasks.register('copy_dependencies', Copy) { + dependsOn create_lib + from configurations.default + into "$projectDir/../docker/lib" + finalizedBy build +} + +tasks.register('create_lib') { + dependsOn delete_lib + finalizedBy copy_dependencies } + +tasks.register('delete_lib', Delete) { + delete "$projectDir/lib" + finalizedBy create_lib +} + +tasks.named('build').configure { + dependsOn copy_dependencies +} + +configurations.implementation { + exclude group: 'jline' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.apache.logging' + exclude group: 'log4j' + exclude group: 'org.ops4j.pax.logging' + exclude group: 'com.google.guava' +} \ No newline at end of file