From fd21339722592fc85f004199cd41326342f73c24 Mon Sep 17 00:00:00 2001 From: Michael Sauter Date: Thu, 12 Jan 2023 17:36:30 +0100 Subject: [PATCH] Remove image wrappers Supply private certificate via secret volume to tasks. Closes #621. --- Makefile | 15 ----- build/package/Dockerfile.gradle-toolset | 9 +-- build/package/Dockerfile.sonar | 5 +- build/package/scripts/build-gradle.sh | 3 +- build/package/scripts/build-npm.sh | 12 ++-- build/package/scripts/build-python.sh | 2 +- ...et-gradle-proxy.sh => configure-gradle.sh} | 19 +++++- build/package/scripts/configure-truststore.sh | 59 ++++++++++++++++++ .../package/scripts/download-aqua-scanner.sh | 44 ++++++------- cmd/sonar/main.go | 59 +++++++++++------- cmd/start/cache.go | 3 +- deploy/README.md | 5 +- deploy/install.sh | 35 +++++++++++ deploy/ods-pipeline/Chart.yaml | 3 - deploy/ods-pipeline/charts/images/Chart.yaml | 23 ------- .../charts/images/docker/Dockerfile.finish | 10 --- .../images/docker/Dockerfile.go-toolset | 10 --- .../images/docker/Dockerfile.gradle-toolset | 10 --- .../charts/images/docker/Dockerfile.helm | 10 --- .../docker/Dockerfile.node16-npm-toolset | 10 --- .../images/docker/Dockerfile.package-image | 24 ------- .../images/docker/Dockerfile.pipeline-manager | 10 --- .../images/docker/Dockerfile.python-toolset | 10 --- .../charts/images/docker/Dockerfile.sonar | 10 --- .../charts/images/docker/Dockerfile.start | 10 --- .../charts/images/templates/_helpers.tpl | 62 ------------------- .../images/templates/bc-ods-finish.yaml | 28 --------- .../images/templates/bc-ods-go-toolset.yaml | 30 --------- .../templates/bc-ods-gradle-toolset.yaml | 30 --------- .../charts/images/templates/bc-ods-helm.yaml | 30 --------- .../templates/bc-ods-node16-npm-toolset.yaml | 30 --------- .../templates/bc-ods-package-image.yaml | 32 ---------- .../templates/bc-ods-pipeline-manager.yaml | 28 --------- .../templates/bc-ods-python-toolset.yaml | 30 --------- .../charts/images/templates/bc-ods-sonar.yaml | 28 --------- .../charts/images/templates/bc-ods-start.yaml | 28 --------- .../images/templates/is-ods-finish.yaml | 8 --- .../images/templates/is-ods-go-toolset.yaml | 10 --- .../templates/is-ods-gradle-toolset.yaml | 10 --- .../charts/images/templates/is-ods-helm.yaml | 10 --- .../templates/is-ods-node16-npm-toolset.yaml | 10 --- .../templates/is-ods-package-image.yaml | 10 --- .../templates/is-ods-pipeline-manager.yaml | 8 --- .../templates/is-ods-python-toolset.yaml | 10 --- .../charts/images/templates/is-ods-sonar.yaml | 8 --- .../charts/images/templates/is-ods-start.yaml | 8 --- .../images/templates/job-start-builds.yaml | 20 ------ deploy/ods-pipeline/charts/images/values.yaml | 2 - .../charts/setup/templates/deployment.yaml | 3 +- .../charts/tasks/templates/_sonar-step.tpl | 11 +++- .../tasks/templates/task-ods-build-go.yaml | 12 +++- .../templates/task-ods-build-gradle.yaml | 12 +++- .../tasks/templates/task-ods-build-npm.yaml | 12 +++- .../templates/task-ods-build-python.yaml | 12 +++- .../tasks/templates/task-ods-deploy-helm.yaml | 12 +++- .../tasks/templates/task-ods-finish.yaml | 15 ++++- .../templates/task-ods-package-image.yaml | 14 ++++- .../tasks/templates/task-ods-start.yaml | 10 ++- .../charts/tasks/values.docs.yaml | 4 -- deploy/ods-pipeline/charts/tasks/values.yaml | 3 + deploy/ods-pipeline/values.kind.yaml | 13 ++-- deploy/ods-pipeline/values.yaml | 46 ++------------ deploy/values.yaml.tmpl | 7 --- docs/creating-an-ods-task.adoc | 2 - docs/installation.adoc | 8 +-- docs/repository-layout.adoc | 2 +- pkg/sonar/client.go | 18 +++--- pkg/sonar/report.go | 2 + pkg/sonar/scan.go | 18 ++++-- pkg/tasktesting/helper.go | 15 ++++- scripts/install-inside-kind.sh | 25 +++++++- scripts/nexus/Dockerfile.arm64 | 2 + .../nexus/{Dockerfile => Dockerfile.x86_64} | 2 + scripts/nginx/nginx-bitbucket.conf | 46 ++++++++++++++ scripts/nginx/nginx-nexus.conf | 46 ++++++++++++++ scripts/nginx/nginx-sonarqube.conf | 46 ++++++++++++++ scripts/run-bitbucket.sh | 19 +++--- scripts/run-nexus.sh | 30 ++++----- scripts/run-sonarqube.sh | 21 ++++--- scripts/run-tls-proxy.sh | 38 ++++++++++++ test/tasks/common_test.go | 4 +- test/tasks/ods-build-gradle_test.go | 4 +- test/testdata/private-cert/README.md | 8 +++ test/testdata/private-cert/openssl.conf | 23 +++++++ test/testdata/private-cert/tls.crt | 24 +++++++ test/testdata/private-cert/tls.key | 28 +++++++++ 86 files changed, 666 insertions(+), 831 deletions(-) rename build/package/scripts/{set-gradle-proxy.sh => configure-gradle.sh} (68%) create mode 100755 build/package/scripts/configure-truststore.sh delete mode 100644 deploy/ods-pipeline/charts/images/Chart.yaml delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.finish delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.helm delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar delete mode 100644 deploy/ods-pipeline/charts/images/docker/Dockerfile.start delete mode 100644 deploy/ods-pipeline/charts/images/templates/_helpers.tpl delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml delete mode 100644 deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml delete mode 100644 deploy/ods-pipeline/charts/images/values.yaml rename scripts/nexus/{Dockerfile => Dockerfile.x86_64} (99%) create mode 100644 scripts/nginx/nginx-bitbucket.conf create mode 100644 scripts/nginx/nginx-nexus.conf create mode 100644 scripts/nginx/nginx-sonarqube.conf create mode 100755 scripts/run-tls-proxy.sh create mode 100644 test/testdata/private-cert/README.md create mode 100644 test/testdata/private-cert/openssl.conf create mode 100644 test/testdata/private-cert/tls.crt create mode 100644 test/testdata/private-cert/tls.key diff --git a/Makefile b/Makefile index 73804286..fd241fb0 100644 --- a/Makefile +++ b/Makefile @@ -144,18 +144,3 @@ ifeq ($(strip $(namespace)),) endif cd scripts && ./install-inside-kind.sh -n $(namespace) .PHONY: deploy - -##@ OpenShift - -start-ods-builds: ## Start builds for each ODS BuildConfig - oc start-build ods-package-image - oc start-build ods-finish - oc start-build ods-go-toolset - oc start-build ods-gradle-toolset - oc start-build ods-helm - oc start-build ods-node16-npm-toolset - oc start-build ods-pipeline-manager - oc start-build ods-python-toolset - oc start-build ods-sonar - oc start-build ods-start -.PHONY: start-ods-builds diff --git a/build/package/Dockerfile.gradle-toolset b/build/package/Dockerfile.gradle-toolset index 8a5fa4aa..b9a670dc 100644 --- a/build/package/Dockerfile.gradle-toolset +++ b/build/package/Dockerfile.gradle-toolset @@ -37,13 +37,8 @@ COPY build/package/scripts/copy-build-if-cached.sh /usr/local/bin/copy-build-if- COPY build/package/scripts/copy-artifacts.sh /usr/local/bin/copy-artifacts COPY build/package/scripts/build-gradle.sh /usr/local/bin/build-gradle COPY build/package/scripts/supply-sonar-project-properties-default.sh /usr/local/bin/supply-sonar-project-properties-default -COPY build/package/scripts/set-gradle-proxy.sh /usr/local/bin/set-gradle-proxy -RUN chmod +x /usr/local/bin/build-gradle && \ - chmod +x /usr/local/bin/cache-build && \ - chmod +x /usr/local/bin/copy-build-if-cached && \ - chmod +x /usr/local/bin/copy-artifacts && \ - chmod +x /usr/local/bin/supply-sonar-project-properties-default && \ - chmod +x /usr/local/bin/set-gradle-proxy +COPY build/package/scripts/configure-gradle.sh /usr/local/bin/configure-gradle +COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore # Add sonar-project.properties COPY build/package/sonar-project.properties.d/gradle.properties /usr/local/default-sonar-project.properties diff --git a/build/package/Dockerfile.sonar b/build/package/Dockerfile.sonar index b28d82b8..677f3723 100644 --- a/build/package/Dockerfile.sonar +++ b/build/package/Dockerfile.sonar @@ -32,13 +32,16 @@ RUN cd /tmp \ FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 ENV SONAR_EDITION="community" \ - OPENSSL_VERSION=1.1 + OPENSSL_VERSION=1.1 \ + JAVA_HOME=/usr/lib/jvm/jre-11 RUN microdnf install --nodocs openssl-${OPENSSL_VERSION}* java-11-openjdk-headless which && microdnf clean all COPY --from=builder /usr/local/bin/sonar /usr/local/bin/sonar COPY --from=builder /usr/local/sonar-scanner-cli /usr/local/sonar-scanner-cli COPY --from=builder /usr/local/cnes/cnesreport.jar /usr/local/cnes/cnesreport.jar +COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore + ENV PATH=/usr/local/sonar-scanner-cli/bin:$PATH USER 1001 diff --git a/build/package/scripts/build-gradle.sh b/build/package/scripts/build-gradle.sh index 6f372479..52e2e94b 100755 --- a/build/package/scripts/build-gradle.sh +++ b/build/package/scripts/build-gradle.sh @@ -43,8 +43,9 @@ echo "Using NEXUS_URL=$NEXUS_URL" echo "Using GRADLE_OPTS=$GRADLE_OPTS" echo "Using GRADLE_USER_HOME=$GRADLE_USER_HOME" echo "Using ARTIFACTS_DIR=$ARTIFACTS_DIR" +mkdir -p "${GRADLE_USER_HOME}" -set-gradle-proxy +configure-gradle echo cd "${WORKING_DIR}" diff --git a/build/package/scripts/build-npm.sh b/build/package/scripts/build-npm.sh index de17a932..ba3bbf3a 100755 --- a/build/package/scripts/build-npm.sh +++ b/build/package/scripts/build-npm.sh @@ -65,17 +65,17 @@ if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -echo "Configuring npm to use Nexus ..." -# Remove the protocol segment from NEXUS_URL -NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') -if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then +echo "Configuring npm to use Nexus (${NEXUS_URL}) ..." +if [ -n "${NEXUS_URL}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then NEXUS_AUTH="$(urlencode "${NEXUS_USERNAME}"):$(urlencode "${NEXUS_PASSWORD}")" npm config set registry="$NEXUS_URL"/repository/npmjs/ npm config set always-auth=true npm config set _auth="$(echo -n "$NEXUS_AUTH" | base64)" npm config set email=no-reply@opendevstack.org - npm config set ca=null - npm config set strict-ssl=false + if [ -f /etc/ssl/certs/private-cert.pem ]; then + echo "Configuring private cert ..." + npm config set cafile=/etc/ssl/certs/private-cert.pem + fi fi; echo "package-*.json checks ..." diff --git a/build/package/scripts/build-python.sh b/build/package/scripts/build-python.sh index 1efd8e0a..a06b0efb 100755 --- a/build/package/scripts/build-python.sh +++ b/build/package/scripts/build-python.sh @@ -55,7 +55,7 @@ if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -echo "Configuring pip to use Nexus ..." +echo "Configuring pip to use Nexus (${NEXUS_URL}) ..." # Remove the protocol segment from NEXUS_URL NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then diff --git a/build/package/scripts/set-gradle-proxy.sh b/build/package/scripts/configure-gradle.sh similarity index 68% rename from build/package/scripts/set-gradle-proxy.sh rename to build/package/scripts/configure-gradle.sh index fddae0c8..095e95c5 100755 --- a/build/package/scripts/set-gradle-proxy.sh +++ b/build/package/scripts/configure-gradle.sh @@ -3,8 +3,20 @@ # This script checks for env variable HTTP_PROXY and adds them to gradle.properties. CONTENT="" -if [[ $HTTP_PROXY != "" ]]; then +if [ -f /etc/ssl/certs/private-cert.pem ]; then + echo "Configuring Gradle to trust private cert ..." + configure-truststore --dest-store ".ods-cache/keystore/cacerts" --dest-storepass password + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + exit 1 + fi + # Configure Gradle to use the modified trust store. + CONTENT+="systemProp.javax.net.ssl.trustStore=.ods-cache/keystore/cacerts\n" + CONTENT+="systemProp.javax.net.ssl.trustStorePassword=password\n" +fi +if [ "${HTTP_PROXY}" != "" ]; then + echo "Configuring Gradle to honor HTTP_PROXY ..." proxy=$(echo "$HTTP_PROXY" | sed -e "s|https://||g" | sed -e "s|http://||g") proxy_hostp=$(echo "$proxy" | cut -d "@" -f2) @@ -32,7 +44,8 @@ if [[ $HTTP_PROXY != "" ]]; then fi fi -if [[ $NO_PROXY != "" ]]; then +if [ "${NO_PROXY}" != "" ]; then + echo "Configuring Gradle to honor NO_PROXY ..." # shellcheck disable=SC2001 noproxy_host=$(echo "$NO_PROXY" | sed -e 's|\,\.|\,\*\.|g') # shellcheck disable=SC2001 @@ -41,6 +54,6 @@ if [[ $NO_PROXY != "" ]]; then CONTENT+="systemProp.https.nonProxyHosts=$noproxy_host\n" fi -if [[ $CONTENT != "" ]]; then +if [ "${CONTENT}" != "" ]; then echo -e "$CONTENT" > "${GRADLE_USER_HOME}/gradle.properties" fi diff --git a/build/package/scripts/configure-truststore.sh b/build/package/scripts/configure-truststore.sh new file mode 100755 index 00000000..5d6af5f0 --- /dev/null +++ b/build/package/scripts/configure-truststore.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -u + +md5_bin="${MD5_BIN:-"md5sum --tag"}" +private_cert="/etc/ssl/certs/private-cert.pem" +src_truststore="${JAVA_HOME}/lib/security/cacerts" +src_pass="changeit" +dest_pass="changeit" + +while [[ "$#" -gt 0 ]]; do + case $1 in + + --src-store) src_truststore="$2"; shift;; + --src-store=*) src_truststore="${1#*=}";; + + --src-storepass) src_pass="$2"; shift;; + --src-storepass=*) src_pass="${1#*=}";; + + --dest-store) dest_truststore="$2"; shift;; + --dest-store=*) dest_truststore="${1#*=}";; + + --dest-storepass) dest_pass="$2"; shift;; + --dest-storepass=*) dest_pass="${1#*=}";; + + --debug) set -x; shift;; + + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +dest_truststore_dir="${dest_truststore%/*}" +mkdir -p "${dest_truststore_dir}" + +md5_private_cert_path="${dest_truststore_dir}/.md5-private-cert" + +if [ -f "${private_cert}" ]; then + md5_private_cert=$(${md5_bin} "${private_cert}") + if [ ! -f "${dest_truststore}" ] || [ "${md5_private_cert}" != "$(cat "${md5_private_cert_path}")" ]; then + echo "Creating truststore with private cert ..." + # Copy global keystone to location where we can write to (hide output containing warnings). + keytool -importkeystore \ + -srckeystore "${src_truststore}" -destkeystore "${dest_truststore}" \ + -deststorepass "${dest_pass}" -srcstorepass "${src_pass}" &> keytool-output.txt + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + cat keytool-output.txt; exit 1 + fi + # Trust private cert (hide output containing warnings). + keytool -importcert -noprompt -trustcacerts \ + -alias private-cert -file "${private_cert}" \ + -keystore "${dest_truststore}" -storepass "${dest_pass}" &> keytool-output.txt + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + cat keytool-output.txt; exit 1 + fi + echo "${md5_private_cert}" > "${md5_private_cert_path}" + fi +else + echo "No private cert found, skipping ..." +fi diff --git a/build/package/scripts/download-aqua-scanner.sh b/build/package/scripts/download-aqua-scanner.sh index fcb6975b..12278b08 100755 --- a/build/package/scripts/download-aqua-scanner.sh +++ b/build/package/scripts/download-aqua-scanner.sh @@ -1,45 +1,39 @@ #!/bin/bash set -eu -md5bin="${MD5_BIN:-"md5sum --tag"}" -debug="${DEBUG:-false}" -aquaScannerUrl="" -binDir=".ods-cache/bin" +md5_bin="${MD5_BIN:-"md5sum --tag"}" +aqua_scanner_url="" +bin_dir=".ods-cache/bin" while [[ "$#" -gt 0 ]]; do case $1 in - --bin-dir) binDir="$2"; shift;; - --bin-dir=*) binDir="${1#*=}";; + --bin-dir) bin_dir="$2"; shift;; + --bin-dir=*) bin_dir="${1#*=}";; - --aqua-scanner-url) aquaScannerUrl="$2"; shift;; - --aqua-scanner-url=*) aquaScannerUrl="${1#*=}";; + --aqua-scanner-url) aqua_scanner_url="$2"; shift;; + --aqua-scanner-url=*) aqua_scanner_url="${1#*=}";; - --debug) debug="$2"; shift;; - --debug=*) debug="${1#*=}";; + --debug) set -x; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -if [ "${debug}" == "true" ]; then - set -x -fi - -aquaScannerPath="${binDir}/aquasec" -md5AquaScannerUrlPath="${binDir}/.md5-aquasec" +aqua_scanner_path="${bin_dir}/aquasec" +md5_aqua_scanner_url_path="${bin_dir}/.md5-aquasec" # Optionally install Aqua scanner. # If the binary already exists and was downloaded from the -# URL given by aquaScannerUrl, skip download. -if [ -n "${aquaScannerUrl}" ] && [ "${aquaScannerUrl}" != "none" ]; then - md5AquaScannerUrl=$(${md5bin} -s "${aquaScannerUrl}") - if [ ! -f "${md5AquaScannerUrlPath}" ] || [ "${md5AquaScannerUrl}" != "$(cat "${md5AquaScannerUrlPath}")" ]; then +# URL given by aqua_scanner_url, skip download. +if [ -n "${aqua_scanner_url}" ] && [ "${aqua_scanner_url}" != "none" ]; then + md5_aqua_scanner_url=$(${md5_bin} -s "${aqua_scanner_url}") + if [ ! -f "${md5_aqua_scanner_url_path}" ] || [ "${md5_aqua_scanner_url}" != "$(cat "${md5_aqua_scanner_url_path}")" ]; then echo 'Installing Aqua scanner...' - curl -v -sSf -L "${aquaScannerUrl}" -o aquasec - mv aquasec "${aquaScannerPath}" - chmod +x "${aquaScannerPath}" - echo "${md5AquaScannerUrl}" > "${md5AquaScannerUrlPath}" + curl -v -sSf -L "${aqua_scanner_url}" -o aquasec + mv aquasec "${aqua_scanner_path}" + chmod +x "${aqua_scanner_path}" + echo "${md5_aqua_scanner_url}" > "${md5_aqua_scanner_url_path}" echo 'Installed Aqua scanner version:' - "${aquaScannerPath}" version + "${aqua_scanner_path}" version fi fi diff --git a/cmd/sonar/main.go b/cmd/sonar/main.go index 425e5476..ba79e98c 100644 --- a/cmd/sonar/main.go +++ b/cmd/sonar/main.go @@ -15,13 +15,26 @@ import ( ) type options struct { - sonarAuthToken string - sonarURL string - sonarEdition string - workingDir string - rootPath string - qualityGate bool - debug bool + sonarAuthToken string + sonarURL string + sonarEdition string + workingDir string + rootPath string + qualityGate bool + trustStore string + trustStorePassword string + debug bool +} + +var defaultOptions = options{ + sonarAuthToken: os.Getenv("SONAR_AUTH_TOKEN"), + sonarURL: os.Getenv("SONAR_URL"), + sonarEdition: os.Getenv("SONAR_EDITION"), + workingDir: ".", + qualityGate: false, + trustStore: "cacerts", + trustStorePassword: "password", + debug: (os.Getenv("DEBUG") == "true"), } func main() { @@ -31,12 +44,14 @@ func main() { } opts := options{rootPath: rootPath} - flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", os.Getenv("SONAR_AUTH_TOKEN"), "sonar-auth-token") - flag.StringVar(&opts.sonarURL, "sonar-url", os.Getenv("SONAR_URL"), "sonar-url") - flag.StringVar(&opts.sonarEdition, "sonar-edition", os.Getenv("SONAR_EDITION"), "sonar-edition") - flag.StringVar(&opts.workingDir, "working-dir", ".", "working directory") - flag.BoolVar(&opts.qualityGate, "quality-gate", false, "require quality gate pass") - flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "debug mode") + flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", defaultOptions.sonarAuthToken, "sonar-auth-token") + flag.StringVar(&opts.sonarURL, "sonar-url", defaultOptions.sonarURL, "sonar-url") + flag.StringVar(&opts.sonarEdition, "sonar-edition", defaultOptions.sonarEdition, "sonar-edition") + flag.StringVar(&opts.workingDir, "working-dir", defaultOptions.workingDir, "working directory") + flag.BoolVar(&opts.qualityGate, "quality-gate", defaultOptions.qualityGate, "require quality gate pass") + flag.StringVar(&opts.trustStore, "truststore", defaultOptions.trustStore, "JKS truststore") + flag.StringVar(&opts.trustStorePassword, "truststore-password", defaultOptions.trustStorePassword, "JKS truststore password") + flag.BoolVar(&opts.debug, "debug", defaultOptions.debug, "debug mode") flag.Parse() var logger logging.LeveledLoggerInterface @@ -58,11 +73,13 @@ func main() { } sonarClient, err := sonar.NewClient(&sonar.ClientConfig{ - APIToken: opts.sonarAuthToken, - BaseURL: opts.sonarURL, - ServerEdition: opts.sonarEdition, - Debug: opts.debug, - Logger: logger, + APIToken: opts.sonarAuthToken, + BaseURL: opts.sonarURL, + ServerEdition: opts.sonarEdition, + TrustStore: opts.trustStore, + TrustStorePassword: opts.trustStorePassword, + Debug: opts.debug, + Logger: logger, }) if err != nil { log.Fatal("sonar client:", err) @@ -96,17 +113,17 @@ func sonarScan( Base: ctxt.PullRequestBase, } } - scanStdout, err := sonarClient.Scan( + err := sonarClient.Scan( sonarProject, ctxt.GitRef, ctxt.GitCommitSHA, prInfo, + os.Stdout, + os.Stdin, ) if err != nil { - logger.Infof(scanStdout) return fmt.Errorf("scan failed: %w", err) } - logger.Infof(scanStdout) logger.Infof("Wait until compute engine task finishes ...") err = waitUntilComputeEngineTaskIsSuccessful(logger, sonarClient) diff --git a/cmd/start/cache.go b/cmd/start/cache.go index b7e6f467..f50ab904 100644 --- a/cmd/start/cache.go +++ b/cmd/start/cache.go @@ -77,7 +77,8 @@ func cleanCache(fsb FileSystemBase, fnRemove RemoveFunc, expirationDays int) err cacheDependenciesPath := filepath.Join(".", odsCacheDependenciesDirName) // To avoid spare files inside the dependency cache which are not supported, - // delete all other areas of the cache. + // delete all unknown folders in there. + // Files outside the dependency cache are left untouched. dirEntryFunc := func(path string, d fs.DirEntry) WalkAndRemovalFlags { if !strings.HasPrefix(path, cacheDependenciesPath) { return 0 // allow files outside the dependency cache area for experimentation diff --git a/deploy/README.md b/deploy/README.md index 30e9b66c..8b375419 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -6,18 +6,15 @@ Manifests in `ods-pipeline` are applied once per project by a project administra ## Subcharts -The `tasks`, `images` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift. +The `tasks` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift. ### Subcharts Contents The resources are defined using Helm: -* `BuildConfig` and `ImageStream` resources (in the `images` subchart) * `Task` resources (in `tasks` subchart) * `ConfigMap` and `Secret` resources used by ODS tasks (in `setup` subchart) * ODS pipeline manager (`Service`/`Deployment`) (in `setup` subchart) -The resources of the `images` subchart are only applicable for OpenShift clusters. The subcharts may individually be enabled or disabled via the umbrella chart's `values.yaml`. - ### Versioning In a KinD cluster there are no versions. Images use the implicit `latest` tag. That makes testing and local development easy. diff --git a/deploy/install.sh b/deploy/install.sh index 40d4ce2d..e6c94bfe 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -19,6 +19,7 @@ BITBUCKET_AUTH="" BITBUCKET_WEBHOOK_SECRET="" NEXUS_AUTH="" SONAR_AUTH="" +PRIVATE_CERT="" # Check prerequisites. KUBECTL_BIN="" @@ -103,6 +104,9 @@ while [[ "$#" -gt 0 ]]; do --sonar-auth) SONAR_AUTH="$2"; shift;; --sonar-auth=*) SONAR_AUTH="${1#*=}";; + --private-cert) PRIVATE_CERT="$2"; shift;; + --private-cert=*) PRIVATE_CERT="${1#*=}";; + *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -173,6 +177,35 @@ installSecret () { fi } +installTLSSecret () { + local secretName="$1" + local privateCert="$2" + local certFile="" + if [ -z "${privateCert}" ]; then + echo "No private cert given, skipping ..." + else + if [ "${privateCert:0:1}" == '/' ] || [ "${privateCert:0:2}" == './' ]; then + if [ ! -f "${privateCert}" ]; then + echo "No cert file exists at ${privateCert}"; exit 1 + fi + certFile="${privateCert}" + else + certFile="private-cert.pem.tmp" + openssl s_client -showcerts -connect "${privateCert}" "${certFile}" + fi + if "${KUBECTL_BIN}" -n "${NAMESPACE}" get "secret/${secretName}" &> /dev/null; then + echo "Re-creating secret ${secretName} ..." + "${KUBECTL_BIN}" -n "${NAMESPACE}" delete secret "${secretName}" + else + echo "Creating secret ${secretName} ..." + fi + "${KUBECTL_BIN}" -n "${NAMESPACE}" create secret generic "${secretName}" \ + --from-file=tls.crt="${certFile}" + rm private-cert.pem.tmp &>/dev/null || true + fi +} + # Manage serviceaccount ... if "${KUBECTL_BIN}" -n "${NAMESPACE}" get serviceaccount/"${SERVICEACCOUNT}" &> /dev/null; then echo "Serviceaccount exists already ..." @@ -233,6 +266,8 @@ else "${SONAR_AUTH}" \ "" \ "Please enter an auth token of a SonarQube user with scan permissions (input will be hidden):" + + installTLSSecret "ods-private-cert" "${PRIVATE_CERT}" fi echo "Installing Helm release ${RELEASE_NAME} ..." diff --git a/deploy/ods-pipeline/Chart.yaml b/deploy/ods-pipeline/Chart.yaml index af375267..e95b68e7 100644 --- a/deploy/ods-pipeline/Chart.yaml +++ b/deploy/ods-pipeline/Chart.yaml @@ -24,9 +24,6 @@ version: 0.8.0 appVersion: "0.8.0" dependencies: - - name: images - version: 0.8.0 - condition: images.enabled - name: setup version: 0.8.0 condition: setup.enabled diff --git a/deploy/ods-pipeline/charts/images/Chart.yaml b/deploy/ods-pipeline/charts/images/Chart.yaml deleted file mode 100644 index 59aa7433..00000000 --- a/deploy/ods-pipeline/charts/images/Chart.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v2 -name: images -description: A Helm chart to setup ODS pipeline images - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -appVersion: "0.8.0" diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish b/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish deleted file mode 100644 index 064b53e2..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.finish +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-finish:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset deleted file mode 100644 index edfa0ce2..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.go-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-go-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset deleted file mode 100644 index 80d83308..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.gradle-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-gradle-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm b/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm deleted file mode 100644 index 7c9bbc56..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.helm +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-helm:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset deleted file mode 100644 index ca28169f..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.node16-npm-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-node16-npm-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image b/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image deleted file mode 100644 index f4e3dd8b..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.package-image +++ /dev/null @@ -1,24 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-package-image:$imageTag - -ARG aquasecScannerUrl -ARG privateCertServer - -USER root - -# Optionally install Aqua scanner. -RUN if [ -z $aquasecScannerUrl ] ; then echo 'Skipping Aqua scanner installation!' ; else echo 'Installing Aqua scanner... getting binary from' $aquasecScannerUrl \ - && curl -v -L $aquasecScannerUrl -o aquasec \ - && mv aquasec /usr/local/bin/ \ - && chmod +x /usr/local/bin/aquasec \ - && echo 'Aqua scanner version:' \ - && aquasec version \ - && echo 'Aqua scanner installation completed!'; \ - fi - -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi - -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager b/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager deleted file mode 100644 index 2fbca7fd..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.pipeline-manager +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-pipeline-manager:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset b/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset deleted file mode 100644 index be3fc1c9..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.python-toolset +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-python-toolset:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar b/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar deleted file mode 100644 index 9f61a243..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.sonar +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-sonar:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/docker/Dockerfile.start b/deploy/ods-pipeline/charts/images/docker/Dockerfile.start deleted file mode 100644 index 1d06d828..00000000 --- a/deploy/ods-pipeline/charts/images/docker/Dockerfile.start +++ /dev/null @@ -1,10 +0,0 @@ -ARG imageTag="latest" - -FROM ghcr.io/opendevstack/ods-pipeline/ods-start:$imageTag - -ARG privateCertServer -USER root -RUN if [ -n "${privateCertServer}" ]; then openssl s_client -showcerts -connect "${privateCertServer}" "/etc/pki/ca-trust/source/anchors/${privateCertServer%:*}.pem" && \ - update-ca-trust; fi -USER 1001 diff --git a/deploy/ods-pipeline/charts/images/templates/_helpers.tpl b/deploy/ods-pipeline/charts/images/templates/_helpers.tpl deleted file mode 100644 index 7ba5edc2..00000000 --- a/deploy/ods-pipeline/charts/images/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "chart.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "chart.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "chart.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "chart.labels" -}} -helm.sh/chart: {{ include "chart.chart" . }} -{{ include "chart.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "chart.selectorLabels" -}} -app.kubernetes.io/name: {{ include "chart.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "chart.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "chart.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml deleted file mode 100644 index 3b45b1fa..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-finish.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-finish - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.finish" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml deleted file mode 100644 index ab0207cb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-go-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGo .Values.goToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-go-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.go-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml deleted file mode 100644 index 5df23721..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-gradle-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGradle .Values.gradleToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-gradle-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.gradle-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml deleted file mode 100644 index dbccf014..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-helm.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.deployHelm .Values.helm}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-helm - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.helm" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml deleted file mode 100644 index 92b9162d..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-node16-npm-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node16NPMToolset}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-node16-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-node16-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.node16-npm-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml deleted file mode 100644 index 94716fc7..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-package-image.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{if or .Values.global.enabledTasks.packageImage .Values.packageImage}} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-package-image - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - - name: aquasecScannerUrl - value: '{{.Values.aquasecScannerUrl}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.package-image" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml deleted file mode 100644 index 1736a639..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-pipeline-manager.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-pipeline-manager - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-pipeline-manager:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.pipeline-manager" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml deleted file mode 100644 index a281cb04..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-python-toolset.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{if or .Values.global.enabledTasks.buildPython .Values.pythonToolset }} -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-python-toolset - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.python-toolset" | nindent 6}} - runPolicy: Serial -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml deleted file mode 100644 index c0e9ba65..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-sonar.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-sonar - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.sonar" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml b/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml deleted file mode 100644 index 72668b51..00000000 --- a/deploy/ods-pipeline/charts/images/templates/bc-ods-start.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: BuildConfig -apiVersion: build.openshift.io/v1 -metadata: - name: ods-start - labels: - {{- include "chart.labels" . | nindent 4}} -spec: - nodeSelector: null - output: - to: - kind: ImageStreamTag - name: 'ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' - resources: {} - successfulBuildsHistoryLimit: 5 - failedBuildsHistoryLimit: 5 - postCommit: {} - strategy: - type: Docker - dockerStrategy: - buildArgs: - - name: imageTag - value: '{{.Values.global.imageTag | default .Chart.AppVersion}}' - - name: privateCertServer - value: '{{.Values.privateCertServer}}' - source: - dockerfile: |- - {{- .Files.Get "docker/Dockerfile.start" | nindent 6}} - runPolicy: Serial diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml deleted file mode 100644 index c42ee665..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-finish.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-finish - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml deleted file mode 100644 index e8a36b04..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-go-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGo .Values.goToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-go-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml deleted file mode 100644 index 6410ac7d..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-gradle-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildGradle .Values.gradleToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-gradle-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml deleted file mode 100644 index 8360dd0e..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-helm.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.deployHelm .Values.helm}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-helm - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml deleted file mode 100644 index 5b0bbaeb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-node16-npm-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildNPM .Values.node16NPMToolset}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-node16-npm-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml deleted file mode 100644 index a88e5d8b..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-package-image.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.packageImage .Values.packageImage}} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-package-image - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml deleted file mode 100644 index e5552010..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-pipeline-manager.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-pipeline-manager - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml deleted file mode 100644 index 3a42d50e..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-python-toolset.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{if or .Values.global.enabledTasks.buildPython .Values.pythonToolset }} -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-python-toolset - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep -{{end}} diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml deleted file mode 100644 index cff36d38..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-sonar.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-sonar - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml b/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml deleted file mode 100644 index 9d9b0445..00000000 --- a/deploy/ods-pipeline/charts/images/templates/is-ods-start.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: ods-start - labels: - {{- include "chart.labels" . | nindent 4}} - annotations: - "helm.sh/resource-policy": keep diff --git a/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml b/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml deleted file mode 100644 index 6a286bdb..00000000 --- a/deploy/ods-pipeline/charts/images/templates/job-start-builds.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{if default true .Values.autoBuild}} -apiVersion: batch/v1 -kind: Job -metadata: - name: ods-start-builds - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "1" - "helm.sh/hook-delete-policy": hook-succeeded,hook-failed -spec: - template: - spec: - serviceAccountName: pipeline - restartPolicy: Never - containers: - - name: post-upgrade-job - image: '{{.Values.autoBuildImage | default "quay.io/openshift/origin-cli:4.10"}}' - command: ["/bin/sh","-c"] - args: ["set -e; oc get bc -l=app.kubernetes.io/name=ods-pipeline -o=name | xargs -I % sh -c 'oc start-build %'"] -{{end}} diff --git a/deploy/ods-pipeline/charts/images/values.yaml b/deploy/ods-pipeline/charts/images/values.yaml deleted file mode 100644 index e8d0e03d..00000000 --- a/deploy/ods-pipeline/charts/images/values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# override name to be consistent with previous, separate chart naming convention(s) -nameOverride: ods-pipeline diff --git a/deploy/ods-pipeline/charts/setup/templates/deployment.yaml b/deploy/ods-pipeline/charts/setup/templates/deployment.yaml index ec1c8747..2f9d4488 100644 --- a/deploy/ods-pipeline/charts/setup/templates/deployment.yaml +++ b/deploy/ods-pipeline/charts/setup/templates/deployment.yaml @@ -17,8 +17,7 @@ spec: containers: - name: pipeline-manager securityContext: {} - image: "{{.Values.pipelineManager.image.registry}}/{{.Values.pipelineManager.image.namespace | default .Release.Namespace}}/{{.Values.pipelineManager.image.repository | default .Chart.Name}}:{{.Values.pipelineManager.image.tag | default .Chart.AppVersion}}" - imagePullPolicy: {{.Values.pipelineManager.image.pullPolicy}} + image: "{{.Values.pipelineManager.imageRepository}}/ods-pipeline-manager:{{.Values.pipelineManager.image.tag | default .Chart.AppVersion}}" ports: - name: http containerPort: 8080 diff --git a/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl b/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl index 5f70d0d7..7ef615ec 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl +++ b/deploy/ods-pipeline/charts/tasks/templates/_sonar-step.tpl @@ -1,7 +1,7 @@ {{- define "sonar-step"}} - name: scan-with-sonar # Image is built from build/package/Dockerfile.sonar. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-sonar:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -31,10 +31,17 @@ echo "Skipping SonarQube analysis" else mkdir -p .ods/artifacts/sonarqube-analysis + configure-truststore --dest-store ".ods-cache/truststore/cacerts" --dest-storepass password # sonar is built from cmd/sonar/main.go. sonar \ -working-dir=$(params.working-dir) \ - -quality-gate=$(params.sonar-quality-gate) + -quality-gate=$(params.sonar-quality-gate) \ + -truststore ".ods-cache/truststore/cacerts" -truststore-password password fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml index 2e055bf7..a09302d5 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-go.yaml @@ -117,7 +117,7 @@ spec: steps: - name: build-go-binary # Image is built from build/package/Dockerfile.go-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-go-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -167,8 +167,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml index 2ed8fe54..4443df1c 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-gradle.yaml @@ -176,7 +176,7 @@ spec: steps: - name: build-gradle-binary # Image is built from build/package/Dockerfile.gradle-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-gradle-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: DEBUG valueFrom: @@ -240,8 +240,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml index a04ba49f..3543020d 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-npm.yaml @@ -134,7 +134,7 @@ spec: steps: - name: build-npm # Image is built from build/package/Dockerfile.node-npm-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-node$(params.node-version)-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-node$(params.node-version)-npm-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -197,8 +197,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml index 2fff994f..1f389c9c 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml @@ -96,7 +96,7 @@ spec: steps: - name: build-python # Image is built from build/package/Dockerfile.python-toolset. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-python-toolset:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -159,8 +159,18 @@ spec: --output-dir=$(params.output-dir) \ --debug=${DEBUG} fi + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) {{- include "sonar-step" . | indent 4}} + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml index 42207d4a..36901184 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-deploy-helm.yaml @@ -101,7 +101,7 @@ spec: steps: - name: helm-upgrade-from-repo # Image is built from build/package/Dockerfile.helm. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-helm:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: DEBUG valueFrom: @@ -119,7 +119,17 @@ spec: -diff-flags="$(params.diff-flags)" \ -upgrade-flags="$(params.upgrade-flags)" \ -age-key-secret=$(params.age-key-secret) + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml index 7a93d691..af60ab55 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-finish.yaml @@ -34,7 +34,7 @@ spec: steps: - name: ods-finish # Image is built from build/package/Dockerfile.finish. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-finish:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -84,14 +84,23 @@ spec: key: debug name: ods-pipeline resources: {} - workingDir: $(workspaces.source.path) script: | # ods-finish is built from cmd/finish/main.go. ods-finish \ -pipeline-run-name=$(params.pipeline-run-name) \ -aggregate-tasks-status=$(params.aggregate-tasks-status) - + volumeMounts: + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt + workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - description: The git repo will be present onto the volume backing this workspace name: source diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml index 02737b82..f344d9fe 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-package-image.yaml @@ -48,7 +48,7 @@ spec: - name: registry description: Image registry to push image to. type: string - default: '{{default .Values.registry .Values.pushRegistry}}' + default: '{{default .Values.pushRegistry}}' - name: image-stream description: Reference of the image stream buildah will produce. If not set, the value of `.ods/component` is used. type: string @@ -91,7 +91,7 @@ spec: steps: - name: build-and-push-image # Image is built from build/package/Dockerfile.package-image. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-package-image:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -155,7 +155,7 @@ spec: download-aqua-scanner \ --aqua-scanner-url=${AQUA_SCANNER_URL} \ - --debug=${DEBUG} + $(case ${DEBUG} in (true) printf -- '--debug'; esac) # ods-package-image is built from cmd/package-image/main.go. ods-package-image \ @@ -172,10 +172,18 @@ spec: volumeMounts: - mountPath: /var/lib/containers name: varlibcontainers + - mountPath: /etc/ssl/certs/private-cert.pem + name: private-cert + readOnly: true + subPath: tls.crt workingDir: $(workspaces.source.path) volumes: - emptyDir: {} name: varlibcontainers + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - name: source {{end}} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml index caed39de..d9818629 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml @@ -124,7 +124,7 @@ spec: steps: - name: ods-start # Image is built from build/package/Dockerfile.start. - image: '{{.Values.registry}}/{{default .Release.Namespace .Values.namespace}}/ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' + image: '{{.Values.imageRepository}}/ods-start:{{.Values.global.imageTag | default .Chart.AppVersion}}' env: - name: HOME value: '/tekton/home' @@ -174,7 +174,6 @@ spec: key: debug name: ods-pipeline resources: {} - workingDir: $(workspaces.source.path) script: | # ods-start is built from cmd/start/main.go. @@ -198,7 +197,12 @@ spec: cp .ods/git-commit-sha $(results.commit.path) echo -n "$(params.url)" > $(results.url.path) - + workingDir: $(workspaces.source.path) + volumes: + - name: private-cert + secret: + secretName: ods-private-cert + optional: true workspaces: - description: The git repo will be cloned onto the volume backing this workspace name: source diff --git a/deploy/ods-pipeline/charts/tasks/values.docs.yaml b/deploy/ods-pipeline/charts/tasks/values.docs.yaml index 585202c0..1640798d 100644 --- a/deploy/ods-pipeline/charts/tasks/values.docs.yaml +++ b/deploy/ods-pipeline/charts/tasks/values.docs.yaml @@ -9,7 +9,3 @@ global: buildNPM: true packageImage: true deployHelm: true - -registry: image-registry.openshift-image-registry.svc:5000 -namespace: ods -pushRegistry: image-registry.openshift-image-registry.svc:5000 diff --git a/deploy/ods-pipeline/charts/tasks/values.yaml b/deploy/ods-pipeline/charts/tasks/values.yaml index e8d0e03d..47b0c55b 100644 --- a/deploy/ods-pipeline/charts/tasks/values.yaml +++ b/deploy/ods-pipeline/charts/tasks/values.yaml @@ -1,2 +1,5 @@ # override name to be consistent with previous, separate chart naming convention(s) nameOverride: ods-pipeline + +imageRepository: ghcr.io/opendevstack/ods-pipeline +pushRegistry: image-registry.openshift-image-registry.svc:5000 diff --git a/deploy/ods-pipeline/values.kind.yaml b/deploy/ods-pipeline/values.kind.yaml index 3338ab51..e214118c 100644 --- a/deploy/ods-pipeline/values.kind.yaml +++ b/deploy/ods-pipeline/values.kind.yaml @@ -16,12 +16,9 @@ setup: tag: "latest" tasks: - # To test with the latest public ods-pipeline images, set global.imageTag to 'latest' and use: - # registry: ghcr.io - # namespace: opendevstack/ods-pipeline - registry: localhost:5000 - namespace: ods - pushRegistry: kind-registry.kind:5000 + # Image repository to pull task images from. + # To test with the latest public ods-pipeline images, set + # global.imageTag to 'latest' and use: 'ghcr.io/opendevstack/ods-pipeline'. + imageRepository: localhost:5000/ods -images: - enabled: false + pushRegistry: kind-registry.kind:5000 diff --git a/deploy/ods-pipeline/values.yaml b/deploy/ods-pipeline/values.yaml index 71e443db..e77a476c 100644 --- a/deploy/ods-pipeline/values.yaml +++ b/deploy/ods-pipeline/values.yaml @@ -20,22 +20,6 @@ global: deployHelm: true -# ####################################### # -# IMAGES CHART CONFIG # -# ####################################### # -images: - # enable chart containing Openshift image streams and build configs - enabled: true - # When using a local/private CA, specify the server (incl. port!) to pull the root CA cert from. - # privateCertServer: 'example.com:443' - # if needed, enable images even though related tasks are disabled, e.g. - # pythonToolset: true - # autoBuild controls whether builds for all BuildConfig resources are started - # automatically after a successful Helm upgrade. autoBuild is enabled by default. - # autoBuild: true - # autoBuildImage allows to override the image used for starting builds. - # autoBuildImage: 'quay.io/openshift/origin-cli:4.9' - # ####################################### # # SETUP CHART CONFIG # @@ -164,21 +148,8 @@ setup: storageSize: '5Gi' # Number of replicas to run for the pipeline manager. replicaCount: 1 - image: - # Image registry from which to pull the pipeline manager container image. - registry: 'image-registry.openshift-image-registry.svc:5000' - # Namespace from which to pull the pipeline manager container image. - # If not given, the image is pulled from the release namespace. - # namespace: 'ods' - # Repository (ImageStream) from which to pull the pipeline manager - # container image. - # If not given, the image name equals the chart name. - repository: 'ods-pipeline-manager' - # Pull policy. - pullPolicy: 'Always' - # Image tag to pull. - # If not given, defaults to the chart appVersion. - # tag: 'latest' + # Repository from which to pull the pipeline manager container image. + imageRepository: ghcr.io/opendevstack/ods-pipeline # Deployment pod resources. Typically these settings should not need to change. resources: limits: @@ -197,21 +168,14 @@ tasks: # enable task definition chart enabled: true - # Registry to pull task images from. - registry: image-registry.openshift-image-registry.svc:5000 - - # ImageStream to pull task images from. - # If images are not located within OpenShift image streams, this maps to - # the organisation under which the images are hosted. - # If not set, defaults to the Helm release namespace. - # namespace: ods + # Image repository to pull task images from. + imageRepository: ghcr.io/opendevstack/ods-pipeline # Custom task prefix (defaults to "ods") # taskPrefix: "foo" # Registry to push images to from ods-package-image task. - # If not set, defaults to the value of "registry". - # pushRegistry: image-registry.openshift-image-registry.svc:5000 + pushRegistry: image-registry.openshift-image-registry.svc:5000 # To define build task specific sidecars and quotas, add resources/sidecar section(s) per task, # e.g. diff --git a/deploy/values.yaml.tmpl b/deploy/values.yaml.tmpl index 1340ec55..bbc1e4b6 100644 --- a/deploy/values.yaml.tmpl +++ b/deploy/values.yaml.tmpl @@ -51,10 +51,3 @@ setup: storageProvisioner: 'kubernetes.io/aws-ebs' # Storage class. On AWS backed clusters, use 'gp2'. storageClassName: 'gp2' - -images: - # URL from which to download aqua-scanner binary. The URL may need to contain basic authentication. - # Make sure that username/password are URL-encoded and that the version matches - # your Aqua server version. - # If you do not want to use Aqua, leave this empty. - aquasecScannerUrl: '' diff --git a/docs/creating-an-ods-task.adoc b/docs/creating-an-ods-task.adoc index e0e9a5e6..ab60b68f 100644 --- a/docs/creating-an-ods-task.adoc +++ b/docs/creating-an-ods-task.adoc @@ -10,8 +10,6 @@ To create a technology-specific Task (e.g. python), the following files should b - [ ] build/package/Dockerfile.python-toolset - The Dockerfile with the dependencies and runtime. - [ ] build/package/scripts/build-python.sh - Bash script to carry out the build, linting, testing operations. -- [ ] deploy/ods-pipeline/charts/images/templates/bc-ods-build-python.yaml - BuildConfig to generate the ods-build-python image. -- [ ] deploy/ods-pipeline/charts/images/templates/is-ods-build-python.yaml - Create ImageStream resource in OpenShift. - [ ] deploy/ods-pipeline/charts/tasks/templates/task-ods-build-python.yaml - The Tekton Task. - [ ] docs/tasks/task-ods-build-python.adoc - To describe the task and its parameters. - [ ] test/tasks/ods-build-python_test.go - A test file to test the behavior of the Tekton Task. diff --git a/docs/installation.adoc b/docs/installation.adoc index 44e79ff1..7df3a8ce 100644 --- a/docs/installation.adoc +++ b/docs/installation.adoc @@ -5,7 +5,7 @@ This guide will show how to install ODS Pipeline in an existing ODS project. It An ODS Pipeline installation consists of the following resources: -* `BuildConfig`, `ImageStream` and `Task` resources +* `Task` resources * `ConfigMap` and `Secret` resources, e.g. holding credentials of centrally installed tools such as Nexus and SonarQube * A pipeline manager, which is creating pipeline runs in response to Bitbucket webhook requests @@ -78,9 +78,7 @@ After you ran the install script, continue with the < "${HELM_GENERATED_VALUES_FILE}" +fi +if [ -f "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-${URL_SUFFIX}" ]; then + BITBUCKET_URL=$(cat "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-${URL_SUFFIX}") + echo " bitbucketUrl: '${BITBUCKET_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" +fi +if [ -f "${ODS_KIND_CREDENTIALS_DIR}/nexus-${URL_SUFFIX}" ]; then + NEXUS_URL=$(cat "${ODS_KIND_CREDENTIALS_DIR}/nexus-${URL_SUFFIX}") + echo " nexusUrl: '${NEXUS_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" +fi +if [ -f "${ODS_KIND_CREDENTIALS_DIR}/sonar-${URL_SUFFIX}" ]; then + SONAR_URL=$(cat "${ODS_KIND_CREDENTIALS_DIR}/sonar-${URL_SUFFIX}") + echo " sonarUrl: '${SONAR_URL}'" >> "${HELM_GENERATED_VALUES_FILE}" +fi + "${ODS_PIPELINE_DIR}"/deploy/install.sh \ --aqua-auth "unavailable:unavailable" \ --aqua-scanner-url "none" \ --bitbucket-auth "${BITBUCKET_AUTH}" \ --nexus-auth "${NEXUS_AUTH}" \ --sonar-auth "${SONAR_AUTH}" \ - -f ./ods-pipeline/values.kind.yaml,./ods-pipeline/values.generated.yaml "$@" + -f "./ods-pipeline/values.kind.yaml,${HELM_GENERATED_VALUES_FILE}" "$@" diff --git a/scripts/nexus/Dockerfile.arm64 b/scripts/nexus/Dockerfile.arm64 index 81d9d703..c1608bc7 100644 --- a/scripts/nexus/Dockerfile.arm64 +++ b/scripts/nexus/Dockerfile.arm64 @@ -1,6 +1,8 @@ FROM klo2k/nexus3:3.30.1-01 USER root + RUN echo "storage.diskCache.diskFreeSpaceLimit=2048" >> /opt/sonatype/nexus/etc/karaf/system.properties && \ echo "nexus.scripts.allowCreation=true" >> /opt/sonatype/nexus/etc/nexus-default.properties + USER nexus diff --git a/scripts/nexus/Dockerfile b/scripts/nexus/Dockerfile.x86_64 similarity index 99% rename from scripts/nexus/Dockerfile rename to scripts/nexus/Dockerfile.x86_64 index 6aa95940..636a7275 100644 --- a/scripts/nexus/Dockerfile +++ b/scripts/nexus/Dockerfile.x86_64 @@ -1,6 +1,8 @@ FROM sonatype/nexus3:3.30.1 USER root + RUN echo "storage.diskCache.diskFreeSpaceLimit=2048" >> /opt/sonatype/nexus/etc/karaf/system.properties && \ echo "nexus.scripts.allowCreation=true" >> /opt/sonatype/nexus/etc/nexus-default.properties + USER nexus diff --git a/scripts/nginx/nginx-bitbucket.conf b/scripts/nginx/nginx-bitbucket.conf new file mode 100644 index 00000000..87016b91 --- /dev/null +++ b/scripts/nginx/nginx-bitbucket.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 8443 ssl; + server_name ods-test-bitbucket-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-bitbucket.kind:7990; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/nginx/nginx-nexus.conf b/scripts/nginx/nginx-nexus.conf new file mode 100644 index 00000000..585408fe --- /dev/null +++ b/scripts/nginx/nginx-nexus.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 8443 ssl; + server_name ods-test-nexus-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-nexus.kind:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/nginx/nginx-sonarqube.conf b/scripts/nginx/nginx-sonarqube.conf new file mode 100644 index 00000000..41bc7ada --- /dev/null +++ b/scripts/nginx/nginx-sonarqube.conf @@ -0,0 +1,46 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 8443 ssl; + server_name ods-test-sonarqube-tls.kind localhost; + ssl_certificate /etc/nginx/tls.crt; + ssl_certificate_key /etc/nginx/tls.key; + location / { + proxy_pass http://ods-test-sonarqube.kind:9000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + } + } +} + + diff --git a/scripts/run-bitbucket.sh b/scripts/run-bitbucket.sh index 91bca6ff..7fcd5f17 100755 --- a/scripts/run-bitbucket.sh +++ b/scripts/run-bitbucket.sh @@ -7,7 +7,8 @@ set -ue SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} -BITBUCKET_SERVER_HOST_PORT="7990" +HOST_HTTP_PORT="7990" +HOST_HTTPS_PORT="7993" BITBUCKET_SERVER_CONTAINER_NAME="ods-test-bitbucket-server" BITBUCKET_SERVER_IMAGE_NAME="atlassian/bitbucket" BITBUCKET_SERVER_IMAGE_TAG="7.6.5" @@ -58,20 +59,24 @@ docker run --name ${BITBUCKET_SERVER_CONTAINER_NAME} \ -e JDBC_PASSWORD=jellyfish \ -e JDBC_URL=jdbc:postgresql://${BITBUCKET_POSTGRES_CONTAINER_NAME}.kind:5432/bitbucket \ -e ELASTICSEARCH_ENABLED=false \ - -d --net kind -p "${BITBUCKET_SERVER_HOST_PORT}:7990" -p 7999:7999 \ + -d --net kind -p "${HOST_HTTP_PORT}:7990" -p 7999:7999 \ "${BITBUCKET_SERVER_IMAGE_NAME}:${BITBUCKET_SERVER_IMAGE_TAG}" if ! "${SCRIPT_DIR}/waitfor-bitbucket.sh" ; then docker logs ${BITBUCKET_SERVER_CONTAINER_NAME} exit 1 fi -BITBUCKET_URL_FULL="http://${BITBUCKET_SERVER_CONTAINER_NAME}.kind:7990" + +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${BITBUCKET_SERVER_CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --host-https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-bitbucket.conf" # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " bitbucketUrl: '${BITBUCKET_URL_FULL}'" >> "${HELM_VALUES_FILE}" mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:8443" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-https" +echo -n "http://${BITBUCKET_SERVER_CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-http" echo -n "admin:NzU0OTk1MjU0NjEzOpzj5hmFNAaawvupxPKpcJlsfNgP" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-auth" echo -n "webhook:s3cr3t" > "${ODS_KIND_CREDENTIALS_DIR}/bitbucket-webhook-secret" diff --git a/scripts/run-nexus.sh b/scripts/run-nexus.sh index 4bba58e1..42624f5a 100755 --- a/scripts/run-nexus.sh +++ b/scripts/run-nexus.sh @@ -5,7 +5,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} INSECURE="" -HOST_PORT="8081" +HOST_HTTP_PORT="8081" +HOST_HTTPS_PORT="8443" ADMIN_USER="admin" ADMIN_PASSWORD="" DEVELOPER_USERNAME="developer" @@ -14,8 +15,8 @@ NEXUS_URL= IMAGE_NAME="ods-test-nexus" CONTAINER_NAME="ods-test-nexus" NEXUS_IMAGE_TAG="3.30.1" -HELM_VALUES_FILE="${ODS_PIPELINE_DIR}/deploy/ods-pipeline/values.generated.yaml" ODS_KIND_CREDENTIALS_DIR="${ODS_PIPELINE_DIR}/deploy/.kind-credentials" +DOCKER_CONTEXT_DIR="${ODS_PIPELINE_DIR}/test/testdata/private-cert" while [[ "$#" -gt 0 ]]; do case $1 in @@ -30,20 +31,16 @@ esac; shift; done echo "Run container using image tag ${NEXUS_IMAGE_TAG}" docker rm -f ${CONTAINER_NAME} || true cd "${SCRIPT_DIR}"/nexus -if [ "$(uname -m)" == "arm64" ]; then - docker build -t ${IMAGE_NAME} -f Dockerfile.arm64 . -else - docker build -t ${IMAGE_NAME} . -fi +docker build -t ${IMAGE_NAME} -f "Dockerfile.$(uname -m)" "${DOCKER_CONTEXT_DIR}" cd - &> /dev/null -docker run -d -p "${HOST_PORT}:8081" --net kind --name ${CONTAINER_NAME} ${IMAGE_NAME} +docker run -d -p "${HOST_HTTP_PORT}:8081" --net kind --name ${CONTAINER_NAME} ${IMAGE_NAME} if ! "${SCRIPT_DIR}/waitfor-nexus.sh" ; then docker logs ${CONTAINER_NAME} exit 1 -fi +fi -NEXUS_URL="http://localhost:${HOST_PORT}" +NEXUS_URL="http://localhost:${HOST_HTTP_PORT}" function runJsonScript { local jsonScriptName=$1 @@ -88,10 +85,15 @@ sed "s|@developer_password@|${DEVELOPER_PASSWORD}|g" "${SCRIPT_DIR}"/nexus/devel runJsonScript "createUser" "-d @${SCRIPT_DIR}/nexus/developer-user-with-password.json" rm "${SCRIPT_DIR}"/nexus/developer-user-with-password.json +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --host-https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-nexus.conf" + # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " nexusUrl: 'http://${CONTAINER_NAME}.kind:8081'" >> "${HELM_VALUES_FILE}" mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:8443" > "${ODS_KIND_CREDENTIALS_DIR}/nexus-https" +echo -n "http://${CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${ODS_KIND_CREDENTIALS_DIR}/nexus-http" echo -n "${DEVELOPER_USERNAME}:${DEVELOPER_PASSWORD}" > "${ODS_KIND_CREDENTIALS_DIR}/nexus-auth" diff --git a/scripts/run-sonarqube.sh b/scripts/run-sonarqube.sh index e6b3568f..c98ced18 100755 --- a/scripts/run-sonarqube.sh +++ b/scripts/run-sonarqube.sh @@ -5,7 +5,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} INSECURE="" -HOST_PORT="9000" +HOST_HTTP_PORT="9000" +HOST_HTTPS_PORT="9443" IMAGE_NAME="ods-test-sonarqube" CONTAINER_NAME="ods-test-sonarqube" SONAR_VERSION="8.4" @@ -13,7 +14,6 @@ SONAR_USERNAME="admin" SONAR_PASSWORD="admin" SONAR_EDITION="community" SONAR_IMAGE_TAG="${SONAR_VERSION}-${SONAR_EDITION}" -HELM_VALUES_FILE="${ODS_PIPELINE_DIR}/deploy/ods-pipeline/values.generated.yaml" ODS_KIND_CREDENTIALS_DIR="${ODS_PIPELINE_DIR}/deploy/.kind-credentials" while [[ "$#" -gt 0 ]]; do @@ -51,9 +51,9 @@ else docker build -t ${IMAGE_NAME}:${SONAR_IMAGE_TAG} --build-arg=from=sonarqube:${SONAR_IMAGE_TAG} . fi cd - &> /dev/null -docker run -d --net kind --name ${CONTAINER_NAME} -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "${HOST_PORT}:9000" ${IMAGE_NAME}:${SONAR_IMAGE_TAG} +docker run -d --net kind --name ${CONTAINER_NAME} -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "${HOST_HTTP_PORT}:9000" ${IMAGE_NAME}:${SONAR_IMAGE_TAG} -SONARQUBE_URL="http://localhost:${HOST_PORT}" +SONARQUBE_URL="http://localhost:${HOST_HTTP_PORT}" if ! "${SCRIPT_DIR}/waitfor-sonarqube.sh" ; then docker logs ${CONTAINER_NAME} exit 1 @@ -66,10 +66,15 @@ tokenResponse=$(curl ${INSECURE} -X POST -sSf --user "${SONAR_USERNAME}:${SONAR_ # {"login":"cd_user","name":"foo","token":"bar","createdAt":"2020-04-22T13:21:54+0000"} token=$(echo "${tokenResponse}" | jq -r .token) +echo "Launch TLS proxy" +TLS_CONTAINER_NAME="${CONTAINER_NAME}-tls" +"${SCRIPT_DIR}/run-tls-proxy.sh" \ + --container-name "${TLS_CONTAINER_NAME}" \ + --host-https-port "${HOST_HTTPS_PORT}" \ + --nginx-conf "nginx-sonarqube.conf" + # Write values / secrets so that it can be picked up by install.sh later. -if [ ! -e "${HELM_VALUES_FILE}" ]; then - echo "setup:" > "${HELM_VALUES_FILE}" -fi -echo " sonarUrl: 'http://${CONTAINER_NAME}.kind:9000'" >> "${HELM_VALUES_FILE}" mkdir -p "${ODS_KIND_CREDENTIALS_DIR}" +echo -n "https://${TLS_CONTAINER_NAME}.kind:8443" > "${ODS_KIND_CREDENTIALS_DIR}/sonar-https" +echo -n "http://${CONTAINER_NAME}.kind:${HOST_HTTP_PORT}" > "${ODS_KIND_CREDENTIALS_DIR}/sonar-http" echo -n ":${token}" > "${ODS_KIND_CREDENTIALS_DIR}/sonar-auth" diff --git a/scripts/run-tls-proxy.sh b/scripts/run-tls-proxy.sh new file mode 100755 index 00000000..97d59d0b --- /dev/null +++ b/scripts/run-tls-proxy.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -ue + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ODS_PIPELINE_DIR=${SCRIPT_DIR%/*} + +HOST_HTTPS_PORT="8443" +CONTAINER_NAME="" +NGINX_CONF="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + + -v|--verbose) set -x;; + + --container-name) CONTAINER_NAME="$2"; shift;; + --container-name=*) CONTAINER_NAME="${1#*=}";; + + --nginx-conf) NGINX_CONF="$2"; shift;; + --nginx-conf=*) NGINX_CONF="${1#*=}";; + + --host-https-port) HOST_HTTPS_PORT="$2"; shift;; + --host-https-port=*) HOST_HTTPS_PORT="${1#*=}";; + + *) echo "Unknown parameter passed: $1"; exit 1;; +esac; shift; done + +NGINX_IMAGE="nginx:alpine-slim" +PRIVATE_CERT_DIR="${ODS_PIPELINE_DIR}/test/testdata/private-cert" +if [ "$(uname -m)" == "arm64" ]; then + NGINX_IMAGE="arm64v8/nginx:alpine-slim" +fi +docker rm -f "${CONTAINER_NAME}" &> /dev/null || true +docker run --name "${CONTAINER_NAME}" \ + -v "${SCRIPT_DIR}/nginx/${NGINX_CONF}:/etc/nginx/nginx.conf:ro" \ + -v "${PRIVATE_CERT_DIR}/tls.crt:/etc/nginx/tls.crt:ro" \ + -v "${PRIVATE_CERT_DIR}/tls.key:/etc/nginx/tls.key:ro" \ + -d --net kind -p "${HOST_HTTPS_PORT}:8443" "${NGINX_IMAGE}" diff --git a/test/tasks/common_test.go b/test/tasks/common_test.go index 4897a78d..039e5e57 100644 --- a/test/tasks/common_test.go +++ b/test/tasks/common_test.go @@ -25,6 +25,7 @@ import ( var alwaysKeepTmpWorkspacesFlag = flag.Bool("always-keep-tmp-workspaces", false, "Whether to keep temporary workspaces from taskruns even when test is successful") var outsideKindFlag = flag.Bool("outside-kind", false, "Whether to continue if not in KinD cluster") var skipSonarQubeFlag = flag.Bool("skip-sonar", false, "Whether to skip SonarQube steps") +var privateCertFlag = flag.Bool("private-cert", false, "Whether to run tests using a private cert") const ( taskKindRef = "Task" @@ -43,7 +44,7 @@ func buildTaskParams(p map[string]string) map[string]string { // requiredServices takes a variable amount of services and removes // SonarQube from the resulting slice if the skipSonarQubeFlag is set. func requiredServices(s ...tasktesting.Service) []tasktesting.Service { - requiredServices := []tasktesting.Service{tasktesting.Nexus} + requiredServices := append([]tasktesting.Service{}, s...) sqIndex := slices.Index(requiredServices, tasktesting.SonarQube) if sqIndex != -1 && *skipSonarQubeFlag { requiredServices = slices.Delete(requiredServices, sqIndex, sqIndex+1) @@ -162,6 +163,7 @@ func runTaskTestCases(t *testing.T, taskName string, requiredServices []tasktest SourceDir: tasktesting.StorageSourceDir, StorageCapacity: tasktesting.StorageCapacity, StorageClassName: tasktesting.StorageClassName, + PrivateCert: *privateCertFlag, }, ) diff --git a/test/tasks/ods-build-gradle_test.go b/test/tasks/ods-build-gradle_test.go index 19c7c649..f650d19a 100644 --- a/test/tasks/ods-build-gradle_test.go +++ b/test/tasks/ods-build-gradle_test.go @@ -48,7 +48,7 @@ func TestTaskODSBuildGradle(t *testing.T) { logContains(ctxt.CollectedLogs, t, "--gradle-options=--no-daemon --stacktrace", "No sonar-project.properties present, using default:", - "Using NEXUS_URL=http://ods-test-nexus.kind:8081", + "ods-test-nexus", "Gradle 7.4.2", "Using GRADLE_OPTS=-Dorg.gradle.jvmargs=-Xmx512M", "Using GRADLE_USER_HOME=/workspace/source/.ods-cache/deps/gradle", @@ -89,7 +89,7 @@ func TestTaskODSBuildGradle(t *testing.T) { logContains(ctxt.CollectedLogs, t, "--gradle-options=--no-daemon --stacktrace", "No sonar-project.properties present, using default:", - "Using NEXUS_URL=http://ods-test-nexus.kind:8081", + "ods-test-nexus", "Gradle 7.4.2", "Using GRADLE_OPTS=-Dorg.gradle.jvmargs=-Xmx512M", "Using GRADLE_USER_HOME=/workspace/source/.ods-cache/deps/gradle", diff --git a/test/testdata/private-cert/README.md b/test/testdata/private-cert/README.md new file mode 100644 index 00000000..eed3b181 --- /dev/null +++ b/test/testdata/private-cert/README.md @@ -0,0 +1,8 @@ +This folder contains a self-signed certificate for `localhost` for testing purposes ONLY. + +The files were created like this: +``` +openssl req -config openssl.conf -new -newkey rsa:2048 -days 3650 -nodes -x509 \ + -keyout tls.key -out tls.crt -extensions req_ext +openssl pkcs12 -export -out tls.p12 -in tls.crt -inkey tls.key -passout pass:password +``` diff --git a/test/testdata/private-cert/openssl.conf b/test/testdata/private-cert/openssl.conf new file mode 100644 index 00000000..3945b16b --- /dev/null +++ b/test/testdata/private-cert/openssl.conf @@ -0,0 +1,23 @@ +[ req ] +default_bits = 2048 +prompt = no +distinguished_name = dn +req_extensions = req_ext +default_md = sha256 + +[ dn ] +C = DE +ST = RLP +L = Ingelheim +O = BI X Digital GmbH +CN = localhost +emailAddress = info@bix-digital.com + +[ req_ext ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = ods-test-nexus-tls.kind +DNS.2 = ods-test-sonarqube-tls.kind +DNS.3 = ods-test-bitbucket-server-tls.kind +DNS.4 = localhost diff --git a/test/testdata/private-cert/tls.crt b/test/testdata/private-cert/tls.crt new file mode 100644 index 00000000..523bd67c --- /dev/null +++ b/test/testdata/private-cert/tls.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID/zCCAuegAwIBAgIJALWuLelTDNHsMA0GCSqGSIb3DQEBCwUAMIGEMQswCQYD +VQQGEwJERTEMMAoGA1UECAwDUkxQMRIwEAYDVQQHDAlJbmdlbGhlaW0xGjAYBgNV +BAoMEUJJIFggRGlnaXRhbCBHbWJIMRIwEAYDVQQDDAlsb2NhbGhvc3QxIzAhBgkq +hkiG9w0BCQEWFGluZm9AYml4LWRpZ2l0YWwuY29tMB4XDTIzMDExMzEzMDcyNloX +DTMzMDExMDEzMDcyNlowgYQxCzAJBgNVBAYTAkRFMQwwCgYDVQQIDANSTFAxEjAQ +BgNVBAcMCUluZ2VsaGVpbTEaMBgGA1UECgwRQkkgWCBEaWdpdGFsIEdtYkgxEjAQ +BgNVBAMMCWxvY2FsaG9zdDEjMCEGCSqGSIb3DQEJARYUaW5mb0BiaXgtZGlnaXRh +bC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCiNLE+Sv6UphwD +X7RcQe9t9wUwJWilnvNoQ16Gon6qUiheW1ZYWfSSB8zh7bCwsIcPboP9Pko/kK/b +8abTgRbW0y1rfz/O+u1AfJf469DnebOIXC146BRTQQ5YPOdF30UpPEreTCnsm/kB +tysYoVJNDLvhKIgtLL5G8JaW3bsHxAL+b2VCUGEWnxnaPWeD5QZNbed6XprfcMci +9rMCABSD4vKz+phTQxUM0SrGtNSrRLZ1ETHjnMEnuUOTcsL6WKz37Ss0liFlO9UN +ojY1lUsW7+8o2IJiYCoSxcCA62KmvhdF/Rom5P2Ro14Di0Q1UneuGLXCHCbk1NDQ +UCeW0LMNAgMBAAGjcjBwMG4GA1UdEQRnMGWCF29kcy10ZXN0LW5leHVzLXRscy5r +aW5kghtvZHMtdGVzdC1zb25hcnF1YmUtdGxzLmtpbmSCIm9kcy10ZXN0LWJpdGJ1 +Y2tldC1zZXJ2ZXItdGxzLmtpbmSCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOC +AQEAbE+92yIOrtNU/BH5zz8KgGZDmtUr1B/gbpub14rTdKUspVEgf8dz95SHzk45 +e9IKNvECt0Awat14HMBW9A99GTkUddeL6xltU1QrrNBJWgahKU//OtoDE1dKE7Re +bPYNLHph6pVAF6TSNvJEaI2AbJgP81FKweStESw8Ov0vyreWiL/jCJ15yrSeJMoA +jk6Bj7JVFHBzCvOgo7/juIo73oAeRlgoeQG6It70n8nhsOrfekXE2a0Z6PTREVRw +zaVmiOJnsuUvjY/Am8V41h65w/+nc4O22+i+Uz36H3KOQlv2v7356bWmCT23cJ8E +bLaDimRBNaSiFJPxtyNg7XqM8Q== +-----END CERTIFICATE----- diff --git a/test/testdata/private-cert/tls.key b/test/testdata/private-cert/tls.key new file mode 100644 index 00000000..6549914b --- /dev/null +++ b/test/testdata/private-cert/tls.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCiNLE+Sv6UphwD +X7RcQe9t9wUwJWilnvNoQ16Gon6qUiheW1ZYWfSSB8zh7bCwsIcPboP9Pko/kK/b +8abTgRbW0y1rfz/O+u1AfJf469DnebOIXC146BRTQQ5YPOdF30UpPEreTCnsm/kB +tysYoVJNDLvhKIgtLL5G8JaW3bsHxAL+b2VCUGEWnxnaPWeD5QZNbed6XprfcMci +9rMCABSD4vKz+phTQxUM0SrGtNSrRLZ1ETHjnMEnuUOTcsL6WKz37Ss0liFlO9UN +ojY1lUsW7+8o2IJiYCoSxcCA62KmvhdF/Rom5P2Ro14Di0Q1UneuGLXCHCbk1NDQ +UCeW0LMNAgMBAAECggEAVI1G/wO0pOPZomCW8ahlGFs1Vx2I6RxsISNM+L1nbX0a +EeVmovNxK1XplLOD8PZz0LV5E98OcQgIw4u5vHKD50N5X5mbN1dRqNXJmjXCVYb4 +8NGr2mSPg75brd9BZwi1UoNdO3dUO0QqjpxeWJ5u+j7zCLFKGtJB9VSgCtBWF/te +Zyssv+ueyGdJxUMRPI5864VrCtB8jWdA7rRkB5EjpON2UJnAZoyZ2Sf0iQQxNu+D +lxjC13u9AaGqhZqihFjfPbelQghwfw3NG6IJZWpkpUnjaM3vb/chU7PkSlZks9B0 ++vsLWUjX9TITxGYPVnR0X426WTtqSffB0w66Ji5l+QKBgQDN8q0gIdc+Gy0iK0iO +92U0UI0OVyneuzhv9XR87izcvGnQGHb9ktowuYk2kyDWwB47TnC9VKQt++THjm6A +VU3qtnT7MeFghxRZNabCMJwOBJ66ZDtAFvMx7YAxko1y/73fMvRIvdiLlGcyh1Cr +6gh9wCpTGAvYa5W93DQxsilKzwKBgQDJoIrFshaPJWylSyJMigseAP9QOcuptTHd +C6/nAREqsem6AoWmeDFgI14nD7zcXkc1qidXrQgJyUw3vg68UfTV1wHax5FouwlX +xfGdHngAa30CVIdBcOt/lKJQwvknKgbxQ1a0gxw7kdHquJl3mkoHsBPPSS/UItBJ +5CxoXDgrYwKBgG67LVSafWV6LTEy/odIA1U2fjEYsn18QmMIIYLOeLaSY34suUWr +wL7z8w+x2bU9hMO0cC+MI95Jkn+tWEKOGx+4zHsHHbvNxSYWD7xXcIClAUcMf0L9 +U/MjVIrS1lTfppIsjmXSFW0u9LGZScm1Qq2CNf+JbU0Gw3Naw4F43nTHAoGAc53H +dY2lOfgv0KjAxjryHQZdYNeDXZE1QpeNCl2mRmE2sR75HHgXeBVPy/5OQd8d3jvv +y2rBDTuMpPoq4Pnt9yEPQzDkxLVbqiUrYvkxvqvkRw3JcVEa+VF1sfHbhQWobXg7 +o876xX3cbUflWJv0j2v8MAS7lF1SKszYWg9ndhkCgYAXSwNRZAYfyu2c9/9JvI3q +OICDjRWCTKqtzWDy7kU7boezCv6tb8VP95o/6ejHri1dUncGScQVc2FTFAthqmqn +flfiM1zEYiIl5YCzmxL26GUwbSulAxJplstXzJDUuF2aTjujbCKFV95j9giT0kwe +oPlaAyM1QM2V5KqSroeydg== +-----END PRIVATE KEY-----