From 1fe18db8c5a66eb79b00596663c0fe3fc6cce630 Mon Sep 17 00:00:00 2001 From: sgayangi Date: Thu, 29 Feb 2024 11:54:47 +0530 Subject: [PATCH] Add Prometheus support to IdP DS --- .../config-deployer/config-ds-configmap.yaml | 4 +-- .../idp/idp-ds/idp-ds-configmap.yaml | 14 ++++++++- .../idp/idp-ds/idp-ds-deployment.yaml | 12 +++++++ .../idp-ds-prometheus-jmx-configmap.yaml | 31 +++++++++++++++++++ helm-charts/values.yaml | 5 +-- helm-charts/values.yaml.template | 8 +++++ .../ballerina/Ballerina.toml | 5 +++ .../ballerina/Dependencies.toml | 18 ++++++++++- idp/idp-domain-service/ballerina/Init.bal | 1 + idp/idp-domain-service/docker/Dockerfile | 1 + idp/idp-domain-service/docker/build.gradle | 29 +++++++++++++++-- idp/idp-domain-service/docker/idp/idp.sh | 13 ++++---- .../docker/config-deployer/config.sh | 2 +- 13 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml diff --git a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml index 679d13b461..002bb2ba2a 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-ds-configmap.yaml @@ -32,8 +32,8 @@ data: metricsReporter="prometheus" [ballerinax.prometheus] port=18008 - {{if .Values.wso2.apk.metrics.configDSBalPort}} - host="{{ .Values.wso2.apk.metrics.configDSBalPort}}" + {{if .Values.wso2.apk.metrics.configDSBalHost}} + host="{{ .Values.wso2.apk.metrics.configDSBalHost}}" {{else}} host="0.0.0.0" {{end}} diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml index 2abf4dcf82..b98a26f779 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-configmap.yaml @@ -25,7 +25,19 @@ data: [ballerina.log] level = "DEBUG" [ballerina.http] - traceLogConsole = true + traceLogConsole = false + {{if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled}} + [ballerina.observe] + metricsEnabled=true + metricsReporter="prometheus" + [ballerinax.prometheus] + port=18011 + {{if .Values.wso2.apk.metrics.idpDSBalHost}} + host="{{ .Values.wso2.apk.metrics.idpDSBalHost}}" + {{else}} + host="0.0.0.0" + {{end}} + {{end}} # Sample configurations [wso2.idp_domain_service.datasourceConfiguration] description = "Database for idp" diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml index d246dd99db..079209c48d 100644 --- a/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml +++ b/helm-charts/templates/idp/idp-ds/idp-ds-deployment.yaml @@ -51,6 +51,12 @@ spec: ports: - containerPort: 9443 protocol: "TCP" + {{ if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} + - containerPort: 18011 + protocol: "TCP" + - containerPort: 18012 + protocol: "TCP" + {{ end }} readinessProbe: httpGet: path: /health @@ -76,6 +82,9 @@ spec: - mountPath: /home/wso2apk/idp/security/wso2carbon.pem name: idp-signing-keystore-volume subPath: wso2carbon.pem + - name: prometheus-jmx-config-volume + mountPath: /tmp/metrics/prometheus-jmx-config-idpds.yml + subPath: prometheus-jmx-config-idpds.yml - name: idp-ds-tls-volume mountPath: /home/wso2apk/idp/security/idp.key {{- if and .Values.idp.idpds.configs .Values.idp.idpds.configs.tls }} @@ -111,6 +120,9 @@ spec: - name: config-toml-volume configMap: name: {{ template "apk-helm.resource.prefix" . }}-idp-ds-configmap + - name: prometheus-jmx-config-volume + configMap: + name: prometheus-jmx-config-idpds - name: idp-signing-keystore-volume secret: secretName: {{ template "apk-helm.resource.prefix" . }}-idp-signing-keystore-secret diff --git a/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml new file mode 100644 index 0000000000..773615b87a --- /dev/null +++ b/helm-charts/templates/idp/idp-ds/idp-ds-prometheus-jmx-configmap.yaml @@ -0,0 +1,31 @@ +# Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-jmx-config-idpds +data: + prometheus-jmx-config-idpds.yml: | + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + # OS related metrics + - pattern: 'java.lang<>(\w+): (.*)' + name: os_$1 + help: Operating System $1 + attrNameSnakeCase: true + type: GAUGE diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index e4738d2641..efa7dfdb10 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -243,8 +243,9 @@ wso2: # certKeyFilename: "" # certFilename: "" metrics: - enabled: true - # configDSBalPort: 0.0.0.0 + enabled: false + # configDSBalHost: 0.0.0.0 + # idpDSBalHost: 0.0.0.0 idp: enabled: true diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 0c5d8331fa..aa0b5cde6d 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -587,6 +587,14 @@ wso2: keepAliveTime: 600 # -- Queue size of the worker threads queueSize: 1000 + metrics: + # -- Enable Prometheus metrics + enabled: false + # -- Configure the host for exposing the config ds ballerina metrics + # configDSBalHost: 0.0.0.0 + # -- Configure the host for exposing the idp ds ballerina metrics. + # idpDSBalHost: 0.0.0.0 + idp: # -- Enable Non production identity server enabled: true diff --git a/idp/idp-domain-service/ballerina/Ballerina.toml b/idp/idp-domain-service/ballerina/Ballerina.toml index 7f9e44b8e9..ac2a6ec1ee 100644 --- a/idp/idp-domain-service/ballerina/Ballerina.toml +++ b/idp/idp-domain-service/ballerina/Ballerina.toml @@ -9,5 +9,10 @@ groupId = "org.postgresql" artifactId = "postgresql" version = "42.2.20" +[[platform.java11.dependency]] +groupId = "io.prometheus.jmx" +artifactId = "jmx_prometheus_javaagent" +version = "0.20.0" + [build-options] observabilityIncluded = true diff --git a/idp/idp-domain-service/ballerina/Dependencies.toml b/idp/idp-domain-service/ballerina/Dependencies.toml index fc72aa1d30..210772f7ea 100644 --- a/idp/idp-domain-service/ballerina/Dependencies.toml +++ b/idp/idp-domain-service/ballerina/Dependencies.toml @@ -375,6 +375,21 @@ modules = [ {org = "ballerinax", packageName = "postgresql", moduleName = "postgresql"} ] +[[package]] +org = "ballerinax" +name = "prometheus" +version = "0.2.0" +dependencies = [ + {org = "ballerina", name = "http"}, + {org = "ballerina", name = "io"}, + {org = "ballerina", name = "lang.string"}, + {org = "ballerina", name = "observe"}, + {org = "ballerina", name = "regex"} +] +modules = [ + {org = "ballerinax", packageName = "prometheus", moduleName = "prometheus"} +] + [[package]] org = "wso2" name = "idp_domain_service" @@ -392,7 +407,8 @@ dependencies = [ {org = "ballerina", name = "url"}, {org = "ballerina", name = "uuid"}, {org = "ballerinai", name = "observe"}, - {org = "ballerinax", name = "postgresql"} + {org = "ballerinax", name = "postgresql"}, + {org = "ballerinax", name = "prometheus"} ] modules = [ {org = "wso2", packageName = "idp_domain_service", moduleName = "idp_domain_service"} diff --git a/idp/idp-domain-service/ballerina/Init.bal b/idp/idp-domain-service/ballerina/Init.bal index b34ebe42ab..78fe6721e2 100644 --- a/idp/idp-domain-service/ballerina/Init.bal +++ b/idp/idp-domain-service/ballerina/Init.bal @@ -20,6 +20,7 @@ import ballerina/log; import ballerina/sql; import ballerina/http; import ballerina/jwt; +import ballerinax/prometheus as _; configurable IDPConfiguration & readonly idpConfiguration = ?; final postgresql:Client|sql:Error dbClient; diff --git a/idp/idp-domain-service/docker/Dockerfile b/idp/idp-domain-service/docker/Dockerfile index 6f8406c822..f07cf74987 100644 --- a/idp/idp-domain-service/docker/Dockerfile +++ b/idp/idp-domain-service/docker/Dockerfile @@ -72,6 +72,7 @@ RUN groupadd --system -g ${USER_GROUP_ID} ${USER_GROUP} \ ADD idp ${USER_HOME}/idp COPY docker-entrypoint.sh ${USER_HOME} +COPY /lib/jmx_prometheus_javaagent-0.20.0.jar /home/${USER}/lib/ RUN chown -R ${USER} ${USER_HOME}/idp \ && chown ${USER} /home/${USER}/docker-entrypoint.sh \ diff --git a/idp/idp-domain-service/docker/build.gradle b/idp/idp-domain-service/docker/build.gradle index 75f812f756..03c2890c1a 100644 --- a/idp/idp-domain-service/docker/build.gradle +++ b/idp/idp-domain-service/docker/build.gradle @@ -18,20 +18,45 @@ apply from: "$projectDir/../../../common-gradle-scripts/docker.gradle" apply from: "$rootDir/../../common-gradle-scripts/copy.gradle" +apply from: "$projectDir/../../../common-gradle-scripts/java.gradle" +dependencies { + implementation libs.prometheus +} + +tasks.register('copy_dependencies', Copy) { + dependsOn create_lib + from configurations.default + into "$projectDir/../docker/lib" + finalizedBy build +} + +tasks.register('create_lib') { + dependsOn delete_lib + finalizedBy copy_dependencies +} + +tasks.register('delete_lib', Delete) { + delete "$projectDir/lib" + finalizedBy create_lib +} + +tasks.named('build').configure { + dependsOn copy_dependencies +} tasks.named('copy_dist').configure{ finalizedBy docker_build } -task build{ +task buildDockerImage{ group 'build' description 'Build docker image' dependsOn 'copy_dist' dependsOn 'docker_build' } -build.configure{ +buildDockerImage.configure{ mustRunAfter(":ballerina:build") dependsOn(":ballerina:build") } \ No newline at end of file diff --git a/idp/idp-domain-service/docker/idp/idp.sh b/idp/idp-domain-service/docker/idp/idp.sh index ffa63a2c11..03cb909688 100755 --- a/idp/idp-domain-service/docker/idp/idp.sh +++ b/idp/idp-domain-service/docker/idp/idp.sh @@ -98,9 +98,10 @@ fi echo "Using Java memory options: $JVM_MEM_OPTS" $JAVACMD \ - $JVM_MEM_OPTS \ - $JAVA_OPTS \ - -classpath "$CLASSPATH" \ - -Djava.io.tmpdir="$IDP_HOME/tmp" \ - -jar idp_domain_service.jar $* - status=$? \ No newline at end of file + $JVM_MEM_OPTS \ + $JAVA_OPTS \ + -classpath "$CLASSPATH" \ + -Djava.io.tmpdir="$IDP_HOME/tmp" \ + -javaagent:/home/wso2apk/jmx_prometheus_javaagent-0.20.0.jar=18012:/tmp/metrics/prometheus-jmx-config-idpds.yml \ + -jar idp_domain_service.jar $* +status=$? diff --git a/runtime/config-deployer-service/docker/config-deployer/config.sh b/runtime/config-deployer-service/docker/config-deployer/config.sh index 9cc37c02d9..ccc254e6af 100644 --- a/runtime/config-deployer-service/docker/config-deployer/config.sh +++ b/runtime/config-deployer-service/docker/config-deployer/config.sh @@ -103,6 +103,6 @@ $JAVACMD \ $JAVA_OPTS \ -classpath "$CLASSPATH" \ -Djava.io.tmpdir="$RUNTIME_HOME/tmp" \ - -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18010:/tmp/metrics/prometheus-jmx-config-configds.yml \ + -javaagent:/home/wso2apk/lib/jmx_prometheus_javaagent-0.20.0.jar=18009:/tmp/metrics/prometheus-jmx-config-configds.yml \ -jar config_deployer_service.jar $* status=$?