KFLUXBUGS-863: introduce results fixes with requisite performance related tuning (dev/stage)

components/monitoring/grafana/base/dashboards/pipeline-service/kustomization.yaml

@@ -2,4 +2,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - https://github.com/openshift-pipelines/pipeline-service/operator/gitops/argocd/grafana/?ref=e52f83e174efb8f308f6e32d1e0fc9f8eb9ed893
+  - https://github.com/openshift-pipelines/pipeline-service/operator/gitops/argocd/grafana/?ref=3b07d3edb07237a26e60b5c9ea7d90cd358f2a2e

components/pipeline-service/development/bump-results-watcher-replicas.yaml

@@ -1,4 +1,4 @@
 - op: replace
   path: /spec/replicas
   # default pipeline-service setting is 1
-  value: 2
+  value: 1

components/pipeline-service/development/config.env

@@ -0,0 +1,33 @@
+DB_USER=
+DB_PASSWORD=
+DB_HOST=
+DB_PORT=5432
+DB_NAME=
+DB_SSLMODE=disable
+DB_SSLROOTCERT=
+DB_ENABLE_AUTO_MIGRATION=true
+SERVER_PORT=8080
+PROMETHEUS_PORT=9090
+PROMETHEUS_HISTOGRAM=true
+TLS_PATH=/etc/tls
+AUTH_DISABLE=false
+AUTH_IMPERSONATE=true
+LOG_LEVEL=info
+LOGS_API=false
+LOGS_TYPE=File
+LOGS_BUFFER_SIZE=5242880
+LOGS_PATH=/logs
+S3_BUCKET_NAME=
+S3_ENDPOINT=
+S3_HOSTNAME_IMMUTABLE=false
+S3_REGION=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_MULTI_PART_SIZE=5242880
+GCS_BUCKET_NAME=
+STORAGE_EMULATOR_HOST=
+K8S_QPS=50
+K8S_BURST=100
+GRPC_WORKER_POOL=50
+PROFILING=true
+PROFILING_PORT=6060

components/pipeline-service/development/kustomization.yaml

@@ -8,10 +8,23 @@ commonAnnotations:
   argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 
 resources:
-  - https://github.com/openshift-pipelines/pipeline-service.git/developer/openshift/gitops/argocd/pipeline-service?ref=e52f83e174efb8f308f6e32d1e0fc9f8eb9ed893
-  - https://github.com/openshift-pipelines/pipeline-service.git/developer/openshift/gitops/argocd/pipeline-service-storage?ref=e52f83e174efb8f308f6e32d1e0fc9f8eb9ed893
+  - https://github.com/openshift-pipelines/pipeline-service.git/developer/openshift/gitops/argocd/pipeline-service?ref=3b07d3edb07237a26e60b5c9ea7d90cd358f2a2e
+  - https://github.com/openshift-pipelines/pipeline-service.git/developer/openshift/gitops/argocd/pipeline-service-storage?ref=3b07d3edb07237a26e60b5c9ea7d90cd358f2a2e
   - ../base/rbac
 
+# generate a new configmap with updated values (logs api, db ssl mode) and replace the default one
+configMapGenerator:
+  - behavior: replace
+    files:
+      - config.env
+    name: api-config
+    options:
+      disableNameSuffixHash: true
+  - behavior: merge
+    name: config-observability
+    literals:
+      - profiling.enable="true"
+
 patches:
   - path: chains-tekton-config-patches.yaml
     target:
@@ -46,3 +59,8 @@ patches:
       kind: Deployment
       namespace: tekton-results
       name: tekton-results-watcher
+  - path: update-results-watcher-performance.yaml
+    target:
+      kind: Deployment
+      namespace: tekton-results
+      name: tekton-results-watcher

components/pipeline-service/development/update-results-watcher-performance.yaml

@@ -0,0 +1,19 @@
+---
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: -threadiness
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "50"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: -qps
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "50"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: -burst
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "100"

components/pipeline-service/staging/base/bump-results-watcher-replicas.yaml

@@ -1,4 +1,4 @@
 - op: replace
   path: /spec/replicas
   # default pipeline-service setting is 1
-  value: 2
+  value: 1

components/pipeline-service/staging/base/config.env

@@ -0,0 +1,33 @@
+DB_USER=
+DB_PASSWORD=
+DB_HOST=
+DB_PORT=5432
+DB_NAME=
+DB_SSLMODE=disable
+DB_SSLROOTCERT=
+DB_ENABLE_AUTO_MIGRATION=true
+SERVER_PORT=8080
+PROMETHEUS_PORT=9090
+PROMETHEUS_HISTOGRAM=true
+TLS_PATH=/etc/tls
+AUTH_DISABLE=false
+AUTH_IMPERSONATE=true
+LOG_LEVEL=info
+LOGS_API=false
+LOGS_TYPE=File
+LOGS_BUFFER_SIZE=5242880
+LOGS_PATH=/logs
+S3_BUCKET_NAME=
+S3_ENDPOINT=
+S3_HOSTNAME_IMMUTABLE=false
+S3_REGION=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_MULTI_PART_SIZE=5242880
+GCS_BUCKET_NAME=
+STORAGE_EMULATOR_HOST=
+K8S_QPS=50
+K8S_BURST=100
+GRPC_WORKER_POOL=50
+PROFILING=true
+PROFILING_PORT=6060

components/pipeline-service/staging/base/kustomization.yaml

@@ -8,12 +8,25 @@ commonAnnotations:
   argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 
 resources:
-  - https://github.com/openshift-pipelines/pipeline-service.git/operator/gitops/argocd/pipeline-service?ref=e52f83e174efb8f308f6e32d1e0fc9f8eb9ed893
+  - https://github.com/openshift-pipelines/pipeline-service.git/operator/gitops/argocd/pipeline-service?ref=3b07d3edb07237a26e60b5c9ea7d90cd358f2a2e
   - pipelines-as-code-secret.yaml
   - ../../base/external-secrets
   - ../../base/testing
   - ../../base/rbac
 
+# generate a new configmap with updated values (logs api, db ssl mode) and replace the default one
+configMapGenerator:
+  - behavior: replace
+    files:
+      - config.env
+    name: api-config
+    options:
+      disableNameSuffixHash: true
+  - behavior: merge
+    name: config-observability
+    literals:
+      - profiling.enable="true"
+
 patches:
   - path: chains-tekton-config-patches.yaml
     target:
@@ -33,20 +46,10 @@ patches:
     target:
       kind: TektonConfig
       name: config
-#  - path: scale-down-exporter.yaml
-#    target:
-#      kind: Deployment
-#      name: pipeline-metrics-exporter
-#      namespace: openshift-pipelines
   - path: update-tekton-config-performance.yaml
     target:
       kind: TektonConfig
       name: config
-  - path: bump-results-watcher-replicas.yaml
-    target:
-      kind: Deployment
-      namespace: tekton-results
-      name: tekton-results-watcher
   - path: update-results-watcher-performance.yaml
     target:
       kind: Deployment

components/pipeline-service/staging/base/update-results-watcher-performance.yaml

@@ -4,4 +4,22 @@
   value: "250m"
 - op: replace
   path: /spec/template/spec/containers/1/resources/limits/cpu
-  value: "250m"
+  value: "250m"
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: -threadiness
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "50"
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: -qps
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "50"
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: -burst
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "100"

components/pipeline-service/staging/stone-stage-p01/deploy.yaml

@@ -897,7 +897,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  config.env: |
+  config.env: |-
     DB_USER=
     DB_PASSWORD=
     DB_HOST=
@@ -926,6 +926,11 @@ data:
     S3_MULTI_PART_SIZE=5242880
     GCS_BUCKET_NAME=
     STORAGE_EMULATOR_HOST=
+    K8S_QPS=50
+    K8S_BURST=100
+    GRPC_WORKER_POOL=50
+    PROFILING=true
+    PROFILING_PORT=6060
 kind: ConfigMap
 metadata:
   annotations:
@@ -1054,6 +1059,7 @@ data:
     metrics.taskrun.duration-type: "histogram"
     metrics.pipelinerun.level: "pipeline"
     metrics.pipelinerun.duration-type: "histogram"
+  profiling.enable: "true"
 kind: ConfigMap
 metadata:
   annotations:
@@ -1157,6 +1163,10 @@ spec:
     port: 9443
     protocol: TCP
     targetPort: metrics
+  - name: profiling
+    port: 6060
+    protocol: TCP
+    targetPort: 6060
   selector:
     app.kubernetes.io/name: tekton-results-api
 ---
@@ -1329,7 +1339,7 @@ spec:
             secretKeyRef:
               key: db.name
               name: tekton-results-database
-        image: quay.io/redhat-appstudio/tekton-results-api:bae7851ff584423503af324200f52cd28ca99116
+        image: quay.io/redhat-appstudio/tekton-results-api:dc54f48f6a1adfde07b923129db33aaa77a61213
         livenessProbe:
           httpGet:
             path: /healthz
@@ -1398,7 +1408,7 @@ metadata:
   name: tekton-results-watcher
   namespace: tekton-results
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: tekton-results-watcher
@@ -1464,6 +1474,12 @@ spec:
         - -check_owner=false
         - -completed_run_grace_period
         - 10m
+        - -threadiness
+        - "50"
+        - -qps
+        - "50"
+        - -burst
+        - "100"
         env:
         - name: SYSTEM_NAMESPACE
           valueFrom:
@@ -1481,7 +1497,7 @@ spec:
           value: tekton-results-api-service.tekton-pipelines.svc.cluster.local:8080
         - name: AUTH_MODE
           value: token
-        image: quay.io/redhat-appstudio/tekton-results-watcher:bae7851ff584423503af324200f52cd28ca99116
+        image: quay.io/redhat-appstudio/tekton-results-watcher:dc54f48f6a1adfde07b923129db33aaa77a61213
         name: watcher
         ports:
         - containerPort: 9090

components/pipeline-service/staging/stone-stg-m01/deploy.yaml

@@ -897,7 +897,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  config.env: |
+  config.env: |-
     DB_USER=
     DB_PASSWORD=
     DB_HOST=
@@ -926,6 +926,11 @@ data:
     S3_MULTI_PART_SIZE=5242880
     GCS_BUCKET_NAME=
     STORAGE_EMULATOR_HOST=
+    K8S_QPS=50
+    K8S_BURST=100
+    GRPC_WORKER_POOL=50
+    PROFILING=true
+    PROFILING_PORT=6060
 kind: ConfigMap
 metadata:
   annotations:
@@ -1054,6 +1059,7 @@ data:
     metrics.taskrun.duration-type: "histogram"
     metrics.pipelinerun.level: "pipeline"
     metrics.pipelinerun.duration-type: "histogram"
+  profiling.enable: "true"
 kind: ConfigMap
 metadata:
   annotations:
@@ -1157,6 +1163,10 @@ spec:
     port: 9443
     protocol: TCP
     targetPort: metrics
+  - name: profiling
+    port: 6060
+    protocol: TCP
+    targetPort: 6060
   selector:
     app.kubernetes.io/name: tekton-results-api
 ---
@@ -1329,7 +1339,7 @@ spec:
             secretKeyRef:
               key: db.name
               name: tekton-results-database
-        image: quay.io/redhat-appstudio/tekton-results-api:bae7851ff584423503af324200f52cd28ca99116
+        image: quay.io/redhat-appstudio/tekton-results-api:dc54f48f6a1adfde07b923129db33aaa77a61213
         livenessProbe:
           httpGet:
             path: /healthz
@@ -1398,7 +1408,7 @@ metadata:
   name: tekton-results-watcher
   namespace: tekton-results
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: tekton-results-watcher
@@ -1464,6 +1474,12 @@ spec:
         - -check_owner=false
         - -completed_run_grace_period
         - 10m
+        - -threadiness
+        - "50"
+        - -qps
+        - "50"
+        - -burst
+        - "100"
         env:
         - name: SYSTEM_NAMESPACE
           valueFrom:
@@ -1481,7 +1497,7 @@ spec:
           value: tekton-results-api-service.tekton-pipelines.svc.cluster.local:8080
         - name: AUTH_MODE
           value: token
-        image: quay.io/redhat-appstudio/tekton-results-watcher:bae7851ff584423503af324200f52cd28ca99116
+        image: quay.io/redhat-appstudio/tekton-results-watcher:dc54f48f6a1adfde07b923129db33aaa77a61213
         name: watcher
         ports:
         - containerPort: 9090