SonarQube Maintenance (LTS update, DB update, SAML setup, Helm migration)

.github/workflows/continuous-integration-workflow.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        version: ['8.9.10.61524'] # 8.9 = LTS
+        version: ['9.9.4'] # 9.9 = LTS
         edition: ['community', 'developer', 'enterprise']
     steps:
       -

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Update api version in ocp templates for image, buildconfig, route and deploymentconfig ([#1072](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1072))
+- SonarQube Maintenance (LTS update, DB update, SAML setup) ([#1211](https://github.com/opendevstack/ods-core/issues/1211))
 
 
 ## [4.3.1] - 2024-02-19

Makefile

@@ -4,9 +4,10 @@ SHELL = /bin/bash
 MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 
-ODS_NAMESPACE := $(shell $(CURDIR)/scripts/get-config-param.sh ODS_NAMESPACE)
-NEXUS_URL := $(shell $(CURDIR)/scripts/get-config-param.sh NEXUS_URL)
-SONARQUBE_URL := $(shell $(CURDIR)/scripts/get-config-param.sh SONARQUBE_URL)
+# Load environment variables from .env file
+include ../ods-configuration/ods-core.env
+export $(shell sed 's/=.*//' ../ods-configuration/ods-core.env)
+
 INSECURE := false
 INSECURE_FLAG :=
 ifeq ($(INSECURE), $(filter $(INSECURE), true yes))
@@ -120,24 +121,19 @@ start-doc-gen-build:
 
 # SONARQUBE
 ## Install or update SonarQube.
-install-sonarqube: apply-sonarqube-build start-sonarqube-build apply-sonarqube-deploy configure-sonarqube
+install-sonarqube: apply-sonarqube-chart start-sonarqube-build configure-sonarqube
 .PHONY: install-sonarqube
 
-## Update OpenShift resources related to the SonarQube image.
-apply-sonarqube-build:
-	cd sonarqube/ocp-config && tailor apply --namespace $(ODS_NAMESPACE) bc,is
+## Apply OpenShift resources related to the SonarQube.
+apply-sonarqube-chart:
+	cd sonarqube/chart && envsubst < values.yaml.template > values.yaml && helm upgrade --install --namespace $(ODS_NAMESPACE) sonarqube . && rm values.yaml
 .PHONY: apply-sonarqube-build
 
 ## Start build of BuildConfig "sonarqube".
 start-sonarqube-build:
 	ocp-scripts/start-and-follow-build.sh --namespace $(ODS_NAMESPACE) --build-config sonarqube
-.PHONY: start-sonarqube-build
-
-## Update OpenShift resources related to the SonarQube service.
-apply-sonarqube-deploy:
-	cd sonarqube/ocp-config && tailor apply --namespace $(ODS_NAMESPACE) --exclude bc,is
 	@echo "Visit $(SONARQUBE_URL)/setup to see if any update actions need to be taken."
-.PHONY: apply-sonarqube-deploy
+.PHONY: start-sonarqube-build
 
 ## Configure SonarQube service.
 configure-sonarqube:
@@ -177,14 +173,14 @@ configure-nexus:
 backup: backup-sonarqube backup-ocp-config
 .PHONY: backup
 
-## Create a backup of OpenShift resources in "cd" namespace.
+## Create a backup of OpenShift resources in "ods" namespace.
 backup-ocp-config:
-	tailor export --namespace $(ODS_NAMESPACE) > backup_cd.yml
+	tailor export --namespace $(ODS_NAMESPACE) > backup_ods.yml
 .PHONY: backup-ocp-config
 
-## Create a backup of the SonarQube database in the current directory.
+## Create a backup of the SonarQube database in backup storage and in the current directory.
 backup-sonarqube:
-	cd sonarqube && ./backup.sh --namespace $(ODS_NAMESPACE) --backup-dir `pwd`
+	cd sonarqube && ./backup.sh --namespace $(ODS_NAMESPACE) --local-copy=true --backup-dir `pwd`
 .PHONY: backup-sonarqube
 
 

configuration-sample/ods-core.env.sample

@@ -85,16 +85,21 @@ SONAR_ADMIN_PASSWORD_B64=changeme
 # Do not change the value manually - the token is created and set automatically during "make configure-sonarqube".
 SONAR_AUTH_TOKEN_B64=changeme
 
-# Application in Crowd used for authentication
-SONAR_CROWD_APPLICATION=sonarqube
-SONAR_CROWD_PASSWORD_B64=changeme
+# Toggle authentication via SAML
+SONAR_AUTH_SAML='true'
+
+# SAML Application used for authentication
+SONAR_SAML_APPLICATION_ID=https://sonarqube-cd.192.168.56.101.nip.io/sonar/saml
+SONAR_SAML_PROVIDER_ID_B64=changeme
+SONAR_SAML_LOGIN_URL_B64=changeme
+SONAR_SAML_CERTIFICATE_B64=changeme
 
 # Postgres DB for SonarQube
 # Image to use for the PostgreSQL database. This needs to be compatible with
 # your SonarQube version, see https://docs.sonarqube.org/latest/requirements/requirements/.
 # Take care when upgrading either database or SQ version.
-# E.g. registry.redhat.io/rhel8/postgresql-12
-SONAR_DATABASE_IMAGE=docker-registry.default.svc:5000/openshift/postgresql:9.6
+# E.g. registry.redhat.io/rhel9/postgresql-15
+SONAR_DATABASE_IMAGE=docker-registry.default.svc:5000/openshift/postgresql:15
 # Connection string for JDBC. Typically this does not need to be changed.
 SONAR_DATABASE_JDBC_URL=jdbc:postgresql://sonarqube-postgresql:5432/sonarqube
 # Database name for SonarQube. Typically this does not need to be changed.
@@ -110,20 +115,10 @@ SONAR_DATABASE_USER=sonarqube
 # - Use "developer", "enterprise" or "datacenter" for commercial editions
 SONAR_EDITION=community
 # SonarQube version.
+# See Dockerhub https://hub.docker.com/_/sonarqube/tags
 # Officially supported is:
-# - 8.9 (LTS release)
-SONAR_VERSION=8.9.10.61524
-https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-8.9.10.61524.zip
-# SonarQube distribution URL.
-# Must be aligned with both SONAR_VERSION and SONAR_EDITION.
-# Check https://binaries.sonarsource.com/ for options.
-SONAR_DISTRIBUTION_URL=https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-8.9.10.61524.zip
-
-# Toggle authentication via Crowd plugin
-SONAR_AUTH_CROWD=true
-
-# All params defined in the template https://github.com/opendevstack/ods-core/blob/master/sonarqube/ocp-config/sonarqube.yml
-# can be overwritten by specifying the param in this file.
+# - 9.9 (LTS release)
+SONAR_VERSION=9.9.4
 
 #########
 # Jira #
@@ -225,8 +220,8 @@ JENKINS_AGENT_BASE_SNYK_DISTRIBUTION_URL=https://github.com/snyk/snyk/releases/d
 # Releases are published at https://download.aquasec.com/scanner
 # Check Aqua versions backward compatibility at https://docs.aquasec.com/docs/version-compatibility-of-components#section-backward-compatibility-across-two-major-versions
 # To Download the aquaSec scanner cli and check their documentaion requires a valid account on aquasec.com
-# Latest tested version is 2022.4.284
-# Example: https://<USER>:<PASSWORD>@download.aquasec.com/scanner/2022.4.284/scannercli
+# Latest tested version is 2022.4.460
+# Example: https://<USER>:<PASSWORD>@download.aquasec.com/scanner/2022.4.460/scannercli
 JENKINS_AGENT_BASE_AQUASEC_SCANNERCLI_URL=
 
 # Repository of shared library

docs/modules/administration/nav.adoc

@@ -1,6 +1,7 @@
 * Administration
 ** xref:administration:installation.adoc[Installation]
 ** Upgrade
+*** xref:administration:helm-migration.adoc[Helm migration]
 *** xref:administration:update-2-to-3.adoc[2.x to 3.x]
 *** xref:administration:update-older.adoc[older]
 ** xref:provisioning-app:configuration.adoc[Provisioning App]

docs/modules/administration/pages/helm-migration.adoc

@@ -0,0 +1,31 @@
+= Migrating from Tailor to Helm
+
+Tailor has been developed for OpenShift 3.11. Back in the days, Helm 2 required the use of a privileged Tiller service and did not work well with OpenShift-specific resources. With Helm 3 and OpenShift 4, this situation has changed.
+
+While Tailor also works in an OpenShift 4 cluster, OpenShift has integrated Helm into its product, and Helm has a huge and growing community. Therefore, it is recommended to use Helm instead of Tailor in an OpenShift 4 cluster.
+
+Tailor is based on OpenShift templates, which define the Kubernetes resources to apply. Helm uses a different templating language / engine, but in the end the templates also describe Kubernetes resources. Therefore, migration effort is relatively low as one only needs to change the syntax of the definition, not the definition itself. Further, there are differences between the CLI of the two tools and not all features of Tailor are available in Helm and vice-versa. Once migration to Helm is complete, it is also recommended to look at the best practices in the Helm community and adopt these.
+
+This document will describe how to adopt resources deployed using Tailor into a Helm release for OpenDevStack installation.
+
+== Templates preparation
+
+There are basically two options how to approach this: you can either adopt your existing OpenShift resources to chart templates, or you can start a clean installation.
+Once the templates have been made ready, you can continue the installation/update following the normal process
+
+=== Option 1: Adopting OpenShift resources
+
+In order to adopt the resources into a Helm release you can use the following script https://github.com/opendevstack/ods-core/blob/master/scripts/tailor2helm.sh
+
+=== Option 2: Clean Installation
+
+In this case you will need to remove all existing resources for the component that has been previously deployed using Tailor, you can check existing resources like this:
+
+[source,sh]
+----
+oc -n ods get all -l app=foo-bar
+----
+
+== Sources 
+
+Check more information about migration from tailor to helm and their differences in https://github.com/opendevstack/tailor/wiki/Migrating-from-Tailor-to-Helm

docs/modules/sonarqube/pages/administration.adoc

@@ -8,9 +8,9 @@ There is an `admin` user which is allowed to change settings, install plugins, e
 
 == Updating SonarQube to a new version
 
-The SonarQube deployment is using an image built in the central ODS namespace. This image is not based on the https://hub.docker.com/_/sonarqube[official SonarQube image on DockerHub], because that requires a specific user to be run under. However, the approach ODS takes is similar to the official image, as can be seen in the https://github.com/opendevstack/ods-core/blob/master/sonarqube/docker/Dockerfile[ODS Dockerfile].
-
-The SonarQube version which is used is determined by which ZIP file is used.
+The SonarQube deployment is using an image built in the central ODS namespace. This image is based on the https://hub.docker.com/_/sonarqube, as can be seen in the https://github.com/opendevstack/ods-core/blob/master/sonarqube/docker/Dockerfile[ODS Dockerfile].
+If the previous version was installed using Tailor, follow the process to migrate from tailor to helm in https://www.opendevstack.org/ods-documentation/opendevstack/5.x/administration/helm-migration.html
+If the previous version was configured to use Atlassian Crowd as Authentication provider, run the script https://github.com/opendevstack/ods-core/tree/master/scripts/migrate-sonar-users.sh in order to migrate current users to use Saml as Authentication provider.
 
 To update SonarQube, the following steps need to be taken:
 
@@ -35,7 +35,7 @@ chmod +x repos.sh
 ./repos.sh --git-ref $GIT_REF --repos="ods-core,ods-configuration" --source-project https://$BITBUCKET_HOST/scm/$BITBUCKET_PROJECT
 ----
 
-The script can also be used in an umbrella directory in which the repositories already exist. Once the repositories are at the desired revision, change `SONAR_VERSION` and `SONAR_DISTRIBUTION_URL` in `ods-configuration/ods-core.env` to your desired version. Commit the change and push it to Bitbucket.
+The script can also be used in an umbrella directory in which the repositories already exist. Once the repositories are at the desired revision, change `SONAR_VERSION` in `ods-configuration/ods-core.env` to your desired version. Commit the change and push it to Bitbucket.
 
 Next, you should ensure that the SonarQube `BuildConfig` resource in the central ODS namespace is up-to-date, then trigger a new build so that a new image gets built. Finally, a new deployment should be rolled out which uses the newly built image. All of this can be done from the `ods-core` repository:
 
@@ -44,4 +44,4 @@ Next, you should ensure that the SonarQube `BuildConfig` resource in the central
 make install-sonarqube
 ----
 
-This target depends on the targets `apply-sonarqube-build`, `start-sonarqube-build`, `apply-sonarqube-deploy` and `configure-sonarqube` which execute the steps described above (and can of course also be executed individually).
+This target depends on the targets `apply-sonarqube-chart`, `start-sonarqube-build` and `configure-sonarqube` which execute the steps described above (and can of course also be executed individually).

docs/modules/sonarqube/pages/index.adoc

@@ -28,11 +28,11 @@ sonar.java.libraries=build/libs
 sonar.junit.reportPaths=build/test-results/test
 ```
 
-The general settings are documented at https://docs.sonarqube.org/display/SONAR/Analysis+Parameters. Plugin-specific parameters can be found in the docs for each plugin, e.g. https://docs.sonarqube.org/display/PLUG/Java+Plugin+and+Bytecode.
+The general settings are documented at https://docs.sonarsource.com/sonarqube/9.9/analyzing-source-code/analysis-parameters.
 
 == Scanning in Jenkins pipelines
 
-There's a prebuilt stage (`odsComponentStageScanWithSonar`) that you can use, see https://www.opendevstack.org/ods-documentation/ods-jenkins-shared-library/latest/index.html for details.
+There's a prebuilt stage (`odsComponentStageScanWithSonar`) that you can use, see https://www.opendevstack.org/ods-documentation/opendevstack/latest/jenkins-shared-library/component-pipeline.html#_odscomponentstagescanwithsonar for details.
 
 == Scanning locally: SonarLint IDE Plugin
 

jenkins/agent-base/Dockerfile.ubi8

@@ -2,8 +2,9 @@ FROM quay.io/openshift/origin-jenkins-agent-base
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ENV SONAR_SCANNER_VERSION=4.7.0.2747 \
-    CNES_REPORT_VERSION=4.1.2 \
+# SONAR_SCANNER_VERSION above 4.8.x require java 17 to run.
+ENV SONAR_SCANNER_VERSION=4.8.1.3023 \
+    CNES_REPORT_VERSION=4.2.0 \
     TAILOR_VERSION=1.3.4 \
     SOPS_VERSION=3.7.3 \
     HELM_VERSION=3.11.3 \

scripts/migrate-sonar-users.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -ue
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ODS_CORE_DIR=${SCRIPT_DIR%/*}
+ODS_CONFIGURATION_DIR="${ODS_CORE_DIR}/../ods-configuration"
+
+echo_done(){
+    echo -e "\033[92mDONE\033[39m: $1"
+}
+
+echo_warn(){
+    echo -e "\033[93mWARN\033[39m: $1"
+}
+
+echo_error(){
+    echo -e "\033[31mERROR\033[39m: $1"
+}
+
+echo_info(){
+    echo -e "\033[94mINFO\033[39m: $1"
+}
+
+
+SONARQUBE_URL=""
+SONAR_ADMIN_USERNAME=""
+SONAR_ADMIN_PASSWORD=""
+INSECURE=""
+
+function usage {
+    printf "Migrate SonarQube users from Atlassian Crowd provider to Saml provider.\n\n"
+    printf "This script will ask interactively for parameters if not in ods-configuraion.\n"
+    printf "However, you can also pass them directly. Usage:\n\n"
+    printf "\t-h|--help\t\tPrint usage\n"
+    printf "\t-v|--verbose\t\tEnable verbose mode\n"
+    printf "\t-i|--insecure\t\tAllow insecure server connections when using SSL\n"
+    printf "\n"
+    printf "\t-s|--sonarqube\t\tSonarQube URL, e.g. 'https://sonarqube.example.com'\n"
+    printf "\t-u|--admin-user\tAdmin user\n"
+    printf "\t-p|--admin-password\tAdmin password\n"
+}
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+
+    -v|--verbose) set -x;;
+
+    -h|--help) usage; exit 0;;
+
+    -i|--insecure) INSECURE="--insecure";;
+
+    -p|--admin-password) SONAR_ADMIN_PASSWORD="$2"; shift;;
+    -p=*|--admin-password=*) SONAR_ADMIN_PASSWORD="${1#*=}";;
+
+    -u|--admin-user) SONAR_ADMIN_USERNAME="$2"; shift;;
+    -u=*|--admin-user=*) SONAR_ADMIN_USERNAME="${1#*=}";;
+
+    -s|--sonarqube) SONARQUBE_URL="$2"; shift;;
+    -s=*|--sonarqube=*) SONARQUBE_URL="${1#*=}";;
+
+    *) echo_error "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+if [ -f "${ODS_CONFIGURATION_DIR}/ods-core.env" ]; then
+
+    if [ -z "${SONARQUBE_URL}" ]; then
+        SONARQUBE_URL=$(../scripts/get-config-param.sh SONARQUBE_URL)
+    fi
+
+    if [ -z "${SONAR_ADMIN_USERNAME}" ]; then
+        SONAR_ADMIN_USERNAME=$(../scripts/get-config-param.sh SONAR_ADMIN_USERNAME)
+    fi
+
+    if [ -z "${SONAR_ADMIN_PASSWORD}" ]; then
+        SONAR_ADMIN_PASSWORD=$(../scripts/get-config-param.sh SONAR_ADMIN_PASSWORD_B64 | base64 -d)
+    fi
+
+fi
+
+Email_list=$( curl ${INSECURE} ${SONAR_URL}/api/users/search -u admin:${SONAR_ADMIN_TOKEN} | jq .users | grep login | grep @ | tr -d '"' | tr -d "," | cut -f2 -d ":" )
+email_list_array=($Email_list)
+
+for email in "${email_list_array[@]}"
+do
+    curl ${INSECURE} -X POST -sSf -u admin:${SONAR_ADMIN_TOKEN} "${SONAR_URL}/api/users/update_identity_provider?newExternalProvider=saml&login=${email}" > /dev/null
+    echo "User ${email} migrated to Saml"
+done