SonarQube Maintenance (LTS update, DB update, SAML setup, Helm migration)

.github/workflows/continuous-integration-workflow.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        version: ['8.9.10.61524'] # 8.9 = LTS
+        version: ['9.9.4'] # 9.9 = LTS
         edition: ['community', 'developer', 'enterprise']
     steps:
       -

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Update api version in ocp templates for image, buildconfig, route and deploymentconfig ([#1072](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1072))
+- SonarQube Maintenance (LTS update, DB update, SAML setup) ([#1211](https://github.com/opendevstack/ods-core/issues/1211))
 
 
 ## [4.3.1] - 2024-02-19

Makefile

@@ -4,9 +4,10 @@ SHELL = /bin/bash
 MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 
-ODS_NAMESPACE := $(shell $(CURDIR)/scripts/get-config-param.sh ODS_NAMESPACE)
-NEXUS_URL := $(shell $(CURDIR)/scripts/get-config-param.sh NEXUS_URL)
-SONARQUBE_URL := $(shell $(CURDIR)/scripts/get-config-param.sh SONARQUBE_URL)
+# Load environment variables from .env file
+include ../ods-configuration/ods-core.env
+export $(shell sed 's/=.*//' ../ods-configuration/ods-core.env)
+
 INSECURE := false
 INSECURE_FLAG :=
 ifeq ($(INSECURE), $(filter $(INSECURE), true yes))
@@ -120,25 +121,19 @@ start-doc-gen-build:
 
 # SONARQUBE
 ## Install or update SonarQube.
-install-sonarqube: apply-sonarqube-build start-sonarqube-build apply-sonarqube-deploy configure-sonarqube
+install-sonarqube: apply-sonarqube-chart start-sonarqube-build configure-sonarqube
 .PHONY: install-sonarqube
 
 ## Update OpenShift resources related to the SonarQube image.
-apply-sonarqube-build:
-	cd sonarqube/ocp-config && tailor apply --namespace $(ODS_NAMESPACE) bc,is
+apply-sonarqube-chart:
+	cd sonarqube/chart && envsubst < values.yaml.template > values.yaml && helm upgrade --install --namespace $(ODS_NAMESPACE) sonarqube . && rm values.yaml
 .PHONY: apply-sonarqube-build
 
 ## Start build of BuildConfig "sonarqube".
 start-sonarqube-build:
 	ocp-scripts/start-and-follow-build.sh --namespace $(ODS_NAMESPACE) --build-config sonarqube
 .PHONY: start-sonarqube-build
 
-## Update OpenShift resources related to the SonarQube service.
-apply-sonarqube-deploy:
-	cd sonarqube/ocp-config && tailor apply --namespace $(ODS_NAMESPACE) --exclude bc,is
-	@echo "Visit $(SONARQUBE_URL)/setup to see if any update actions need to be taken."
-.PHONY: apply-sonarqube-deploy
-
 ## Configure SonarQube service.
 configure-sonarqube:
 	cd sonarqube && ./configure.sh --sonarqube=$(SONARQUBE_URL) $(INSECURE_FLAG)

configuration-sample/ods-core.env.sample

@@ -85,16 +85,21 @@ SONAR_ADMIN_PASSWORD_B64=changeme
 # Do not change the value manually - the token is created and set automatically during "make configure-sonarqube".
 SONAR_AUTH_TOKEN_B64=changeme
 
-# Application in Crowd used for authentication
-SONAR_CROWD_APPLICATION=sonarqube
-SONAR_CROWD_PASSWORD_B64=changeme
+# Toggle authentication via SAML
+SONAR_AUTH_SAML='true'
+
+# SAML Application used for authentication
+SONAR_SAML_APPLICATION_ID=https://sonarqube-cd.192.168.56.101.nip.io/sonar/saml
+SONAR_SAML_PROVIDER_ID_B64=changeme
+SONAR_SAML_LOGIN_URL_B64=changeme
+SONAR_SAML_CERTIFICATE_B64=changeme
 
 # Postgres DB for SonarQube
 # Image to use for the PostgreSQL database. This needs to be compatible with
 # your SonarQube version, see https://docs.sonarqube.org/latest/requirements/requirements/.
 # Take care when upgrading either database or SQ version.
-# E.g. registry.redhat.io/rhel8/postgresql-12
-SONAR_DATABASE_IMAGE=docker-registry.default.svc:5000/openshift/postgresql:9.6
+# E.g. registry.redhat.io/rhel9/postgresql-15
+SONAR_DATABASE_IMAGE=docker-registry.default.svc:5000/openshift/postgresql:15
 # Connection string for JDBC. Typically this does not need to be changed.
 SONAR_DATABASE_JDBC_URL=jdbc:postgresql://sonarqube-postgresql:5432/sonarqube
 # Database name for SonarQube. Typically this does not need to be changed.
@@ -111,19 +116,8 @@ SONAR_DATABASE_USER=sonarqube
 SONAR_EDITION=community
 # SonarQube version.
 # Officially supported is:
-# - 8.9 (LTS release)
-SONAR_VERSION=8.9.10.61524
-https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-8.9.10.61524.zip
-# SonarQube distribution URL.
-# Must be aligned with both SONAR_VERSION and SONAR_EDITION.
-# Check https://binaries.sonarsource.com/ for options.
-SONAR_DISTRIBUTION_URL=https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-8.9.10.61524.zip
-
-# Toggle authentication via Crowd plugin
-SONAR_AUTH_CROWD=true
-
-# All params defined in the template https://github.com/opendevstack/ods-core/blob/master/sonarqube/ocp-config/sonarqube.yml
-# can be overwritten by specifying the param in this file.
+# - 9.9 (LTS release)
+SONAR_VERSION=9.9.4
 
 #########
 # Jira #
@@ -225,8 +219,8 @@ JENKINS_AGENT_BASE_SNYK_DISTRIBUTION_URL=https://github.com/snyk/snyk/releases/d
 # Releases are published at https://download.aquasec.com/scanner
 # Check Aqua versions backward compatibility at https://docs.aquasec.com/docs/version-compatibility-of-components#section-backward-compatibility-across-two-major-versions
 # To Download the aquaSec scanner cli and check their documentaion requires a valid account on aquasec.com
-# Latest tested version is 2022.4.284
-# Example: https://<USER>:<PASSWORD>@download.aquasec.com/scanner/2022.4.284/scannercli
+# Latest tested version is 2022.4.460
+# Example: https://<USER>:<PASSWORD>@download.aquasec.com/scanner/2022.4.460/scannercli
 JENKINS_AGENT_BASE_AQUASEC_SCANNERCLI_URL=
 
 # Repository of shared library

docs/modules/administration/nav.adoc

@@ -1,6 +1,7 @@
 * Administration
 ** xref:administration:installation.adoc[Installation]
 ** Upgrade
+*** xref:administration:helm-migration.adoc[Helm migration]
 *** xref:administration:update-2-to-3.adoc[2.x to 3.x]
 *** xref:administration:update-older.adoc[older]
 ** xref:provisioning-app:configuration.adoc[Provisioning App]

docs/modules/administration/pages/helm-migration.adoc

@@ -0,0 +1,31 @@
+= Migrating from Tailor to Helm
+
+Tailor has been developed for OpenShift 3.11. Back in the days, Helm 2 required the use of a privileged Tiller service and did not work well with OpenShift-specific resources. With Helm 3 and OpenShift 4, this situation as changed.
+
+While Tailor also works in an OpenShift 4 cluster, OpenShift has integrated Helm into its product, and Helm has a huge and growing community. Therefore, it is recommended to use Helm instead of Tailor in an OpenShift 4 cluster.
+
+This document will describe how to migrate from Tailor to Helm for OpenDevStack installation.
+
+Tailor is based on OpenShift templates, which define the Kubernetes resources to apply. Helm uses a different templating language / engine, but in the end the templates also describe Kubernetes resources. Therefore, migration effort is relatively low as one only needs to change the syntax of the definition, not the definition itself. Further, there are differences between the CLI of the two tools and not all features of Tailor are available in Helm and vice-versa. Once migration to Helm is complete, it is also recommended to look at the best practices in the Helm community and adopt these.
+
+== Templates preparation
+
+There are basically two options how to approach this: you can either adopt your existing OpenShift resources to chart templates, or you can start a clean installation.
+Once the templates have been made ready, you can continue the installation/update following the normal process
+
+=== Option 1: Adopting OpenShift resources
+
+In order to adopt the resources into a Helm release you can use the following script https://github.com/opendevstack/ods-core/blob/master/scripts/tailor2helm.sh
+
+=== Option 2: Clean Installation
+
+In this case you will need to remove all existing resources for the component that has been previously deployed using Tailor, you can check existing resources like this:
+
+[source,sh]
+----
+oc -n ods get all -l app=foo-bar
+----
+
+== Sources 
+
+Check more information about migration from tailor to helm and their differences in https://github.com/opendevstack/tailor/wiki/Migrating-from-Tailor-to-Helm

docs/modules/sonarqube/pages/administration.adoc

@@ -8,9 +8,8 @@ There is an `admin` user which is allowed to change settings, install plugins, e
 
 == Updating SonarQube to a new version
 
-The SonarQube deployment is using an image built in the central ODS namespace. This image is not based on the https://hub.docker.com/_/sonarqube[official SonarQube image on DockerHub], because that requires a specific user to be run under. However, the approach ODS takes is similar to the official image, as can be seen in the https://github.com/opendevstack/ods-core/blob/master/sonarqube/docker/Dockerfile[ODS Dockerfile].
-
-The SonarQube version which is used is determined by which ZIP file is used.
+The SonarQube deployment is using an image built in the central ODS namespace. This image is based on the https://hub.docker.com/_/sonarqube, as can be seen in the https://github.com/opendevstack/ods-core/blob/master/sonarqube/docker/Dockerfile[ODS Dockerfile].
+If the previous version was installed using Tailor, follow the process to migrate from tailor to helm in https://www.opendevstack.org/ods-documentation/opendevstack/5.x/administration/helm-migration.html
 
 To update SonarQube, the following steps need to be taken:
 
@@ -35,7 +34,7 @@ chmod +x repos.sh
 ./repos.sh --git-ref $GIT_REF --repos="ods-core,ods-configuration" --source-project https://$BITBUCKET_HOST/scm/$BITBUCKET_PROJECT
 ----
 
-The script can also be used in an umbrella directory in which the repositories already exist. Once the repositories are at the desired revision, change `SONAR_VERSION` and `SONAR_DISTRIBUTION_URL` in `ods-configuration/ods-core.env` to your desired version. Commit the change and push it to Bitbucket.
+The script can also be used in an umbrella directory in which the repositories already exist. Once the repositories are at the desired revision, change `SONAR_VERSION` in `ods-configuration/ods-core.env` to your desired version. Commit the change and push it to Bitbucket.
 
 Next, you should ensure that the SonarQube `BuildConfig` resource in the central ODS namespace is up-to-date, then trigger a new build so that a new image gets built. Finally, a new deployment should be rolled out which uses the newly built image. All of this can be done from the `ods-core` repository:
 
@@ -44,4 +43,4 @@ Next, you should ensure that the SonarQube `BuildConfig` resource in the central
 make install-sonarqube
 ----
 
-This target depends on the targets `apply-sonarqube-build`, `start-sonarqube-build`, `apply-sonarqube-deploy` and `configure-sonarqube` which execute the steps described above (and can of course also be executed individually).
+This target depends on the targets `apply-sonarqube-chart`, `start-sonarqube-build` and `configure-sonarqube` which execute the steps described above (and can of course also be executed individually).

docs/modules/sonarqube/pages/index.adoc

@@ -28,11 +28,11 @@ sonar.java.libraries=build/libs
 sonar.junit.reportPaths=build/test-results/test
 ```
 
-The general settings are documented at https://docs.sonarqube.org/display/SONAR/Analysis+Parameters. Plugin-specific parameters can be found in the docs for each plugin, e.g. https://docs.sonarqube.org/display/PLUG/Java+Plugin+and+Bytecode.
+The general settings are documented at https://docs.sonarsource.com/sonarqube/9.9/analyzing-source-code/analysis-parameters.
 
 == Scanning in Jenkins pipelines
 
-There's a prebuilt stage (`odsComponentStageScanWithSonar`) that you can use, see https://www.opendevstack.org/ods-documentation/ods-jenkins-shared-library/latest/index.html for details.
+There's a prebuilt stage (`odsComponentStageScanWithSonar`) that you can use, see https://www.opendevstack.org/ods-documentation/opendevstack/latest/jenkins-shared-library/component-pipeline.html#_odscomponentstagescanwithsonar for details.
 
 == Scanning locally: SonarLint IDE Plugin
 

jenkins/agent-base/Dockerfile.ubi8

@@ -2,8 +2,9 @@ FROM quay.io/openshift/origin-jenkins-agent-base
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ENV SONAR_SCANNER_VERSION=4.7.0.2747 \
-    CNES_REPORT_VERSION=4.1.2 \
+# SONAR_SCANNER_VERSION above 4.8.x require java 17 to run.
+ENV SONAR_SCANNER_VERSION=4.8.1.3023 \
+    CNES_REPORT_VERSION=4.2.0 \
     TAILOR_VERSION=1.3.4 \
     SOPS_VERSION=3.7.3 \
     HELM_VERSION=3.11.3 \

scripts/tailor2helm.sh

@@ -0,0 +1,83 @@
+#bin/bash
+
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ODS_CORE_DIR=${SCRIPT_DIR%/*}
+ODS_CONFIGURATION_DIR="${ODS_CORE_DIR}/../ods-configuration"
+
+echo_done(){
+    echo -e "\033[92mDONE\033[39m: $1"
+}
+
+echo_warn(){
+    echo -e "\033[93mWARN\033[39m: $1"
+}
+
+echo_error(){
+    echo -e "\033[31mERROR\033[39m: $1"
+}
+
+echo_info(){
+    echo -e "\033[94mINFO\033[39m: $1"
+}
+
+LABEL=""
+RELEASE=""
+NAMESPACE=""
+
+function usage {
+    printf "Adopt Tailor resources into Helm release.\n\n"
+    printf "This script will ask interactively for parameters by default.\n"
+    printf "Namespace is optional, but will be read from ods-core.env if available.\n"
+    printf "However, you can also pass them directly. Usage:\n\n"
+    printf "\t-h|--help\t\tPrint usage\n"
+    printf "\t-v|--verbose\t\tEnable verbose mode\n"
+    printf "\n"
+    printf "\t-l|--label\t\tResources label, e.g. 'app=component'\n"
+    printf "\t-r|--release\t\tHelm relese\n"
+    printf "\t-n|--namespace\t\tODS namespace\n"    
+}
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+
+    -v|--verbose) set -x;;
+
+    -h|--help) usage; exit 0;;
+
+    -l|--label) LABEL="$2"; shift;;
+    -l=*|--label=*) LABEL="${1#*=}";;
+
+    -r|--release) RELEASE="$2"; shift;;
+    -r=*|--release=*) RELEASE="${1#*=}";;
+
+    -o|--namespace) NAMESPACE="$2"; shift;;
+    -o=*|--namespace=*) NAMESPACE="${1#*=}";;
+
+  *) echo_error "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+if [ -f "${ODS_CONFIGURATION_DIR}/ods-core.env" ] && [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=$(../scripts/get-config-param.sh ODS_NAMESPACE)
+fi
+
+if [ -z "${LABEL}" ]; then
+    read -r -e -p "Enter label of the resources to adopt: " input
+    LABEL="${input}"
+fi
+
+if [ -z "${RELEASE}" ]; then
+    read -r -e -p "Enter helm release name: " input
+    RELEASE="${input}"
+fi
+
+KINDS='ImageStream,BuildConfig,Service,DeploymentConfig,Deployment,Route,ConfigMap,Secret,PersistentVolumeClaim,ServiceAccount,RoleBinding'
+RESOURCES=$(oc -n $NAMESPACE get $KINDS -l $LABEL -o template='{{range .items}}{{.kind}}/{{.metadata.name}} {{end}}')
+
+for RESOURCE in $RESOURCES; do
+    echo "Adopting $RESOURCE ..."
+    oc -n $NAMESPACE annotate $RESOURCE meta.helm.sh/release-name=$RELEASE
+    oc -n $NAMESPACE annotate $RESOURCE meta.helm.sh/release-namespace=$NAMESPACE
+    oc -n $NAMESPACE label $RESOURCE app.kubernetes.io/managed-by=Helm
+done

sonarqube/README.md

@@ -2,16 +2,15 @@
 
 SonarQube is an open source platform developed by SonarSource for continuous inspection of code quality to perform automatic reviews with static analysis of code to detect bugs, code smells, and security vulnerabilities.
 
-This repo contains the build recipe for a central SonarQube instance to which all Jenkins instances send reports to. The remainder of this readme describes how that setup is done.
-If you are looking for instructions how to analyse your repositories, please see [USAGE.md](https://github.com/opendevstack/ods-core/blob/master/USAGE.md).
+This repo contains the build recipe for a central SonarQube instance to which all Jenkins instances send reports to.
 
 ## Setup
 
-The OpenShift templates are located in `ocp-config` and can be compared with the OC cluster using [tailor](https://github.com/opendevstack/tailor). For example, run `cd ocp-config && tailor status` to see if there is any drift between current and desired state.
+The OpenShift templates are located in `chart` and can be compared with the OC cluster using [helm](https://github.com/helm/helm). For example, run `cd chart && helm secrets diff upgrade` to see if there is any drift between current and desired state.
 
 ## Administration
 
-There is an `admin` user which is allowed to change settings, install plugins, etc. The password is located in the OC project `cd`, under the `sonarqube-app` secrets.
+There is an `admin` user which is allowed to change settings, install plugins, etc. The password is located in the OC project `ods`, under the `sonarqube-app` secrets.
 
 ## Building a new image
 

sonarqube/backup.sh

@@ -1,15 +1,23 @@
 #!/bin/bash
 set -eu
 
-NAMESPACE="ods"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ODS_CORE_DIR=${SCRIPT_DIR%/*}
+ODS_CONFIGURATION_DIR="${ODS_CORE_DIR}/../ods-configuration"
+
+NAMESPACE=""
 BACKUP_DIR="."
+PROGRESS=false
+LOCAL=false
 
 function usage {
-  printf "usage: %s [options]\n" "$0"
+  printf "usage: %s [options]\n" "$0" 
   printf "\t-h|--help\tPrints the usage\n"
   printf "\t-v|--verbose\tVerbose output\n"
   printf "\t-b|--backup-dir\tLocation of backup directory\n"
-  printf "\t-n|--namespace\tNamespace (defaults to '%s')\n" "${NAMESPACE}"
+  printf "\t-n|--namespace\tNamespace, optional param as it will be read from ods-core.env if available\n"
+  printf "\t-p|--progress\tShow progress during transfer (defaults to '%s')\n" "${PROGRESS}"
+  printf "\t-l|--local-copy\tCopy the database dump in local (defaults to '%s')\n" "${LOCAL}"
 }
 
 while [[ "$#" -gt 0 ]]; do case $1 in
@@ -23,22 +31,32 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -n=*|--namespace=*) NAMESPACE="${1#*=}";;
   -n|--namespace) NAMESPACE="$2"; shift;;
 
+  -p=*|--progress=*) PROGRESS="${1#*=}";;
+  -p|--progress) PROGRESS="$2"; shift;;
+
+  -l=*|--local-copy=*) LOCAL="${1#*=}";;
+  -l|--local-copy) LOCAL="$2"; shift;;
+
   *) echo "Unknown parameter passed: $1"; usage; exit 1;;
 esac; shift; done
 
+if [ -f "${ODS_CONFIGURATION_DIR}/ods-core.env" ] && [ -z "${NAMESPACE}" ]; then
+    NAMESPACE=$(../scripts/get-config-param.sh ODS_NAMESPACE)
+fi
+
 if ! oc whoami > /dev/null; then
   echo "You need to log into OpenShift first"
   exit 1
 fi
 
 # Dump database
-destinationFile="${BACKUP_DIR}/sonarqube.sql"
 podWithPrefix=$(oc get pods -n "${NAMESPACE}" --selector name=sonarqube-postgresql --no-headers -o name)
 pod=${podWithPrefix#"pod/"}
-oc rsh -n "${NAMESPACE}" "pod/${pod}" bash -c "mkdir -p /var/lib/pgsql/backup && pg_dump sonarqube > /var/lib/pgsql/backup/sonarqube.sql"
-# Copy export
-oc -n "${NAMESPACE}" cp "${pod}:/var/lib/pgsql/backup/sonarqube.sql" "${destinationFile}"
-# Delete export in pod
-oc rsh -n "${NAMESPACE}" "pod/${pod}" bash -c "rm /var/lib/pgsql/backup/sonarqube.sql"
+oc rsh -n "${NAMESPACE}" "pod/${pod}" bash -c "pg_dump sonarqube > /var/lib/pgsql/backups/sonarqube.sql"
+echo "Database 'sonarqube' dumped and stored in /var/lib/pgsql/backups/sonarqube.sql."
 
-echo "Database 'sonarqube' backed up to ${destinationFile}."
+if [ "${LOCAL}" == true ]; then
+# Copy export locally
+oc -n "${NAMESPACE}" rsync --progress="${PROGRESS}" "${pod}:/var/lib/pgsql/backups/sonarqube.sql" "${BACKUP_DIR}"
+echo "Database 'sonarqube' backed up to ${BACKUP_DIR}."
+fi