Skip to content

Commit

Permalink
Remove image wrappers
Browse files Browse the repository at this point in the history
Supply private certificate via secret volume to tasks.

Closes #621.
  • Loading branch information
michaelsauter committed Jan 18, 2023
1 parent baae2b8 commit fd21339
Show file tree
Hide file tree
Showing 86 changed files with 666 additions and 831 deletions.
15 changes: 0 additions & 15 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -144,18 +144,3 @@ ifeq ($(strip $(namespace)),)
endif
cd scripts && ./install-inside-kind.sh -n $(namespace)
.PHONY: deploy

##@ OpenShift

start-ods-builds: ## Start builds for each ODS BuildConfig
oc start-build ods-package-image
oc start-build ods-finish
oc start-build ods-go-toolset
oc start-build ods-gradle-toolset
oc start-build ods-helm
oc start-build ods-node16-npm-toolset
oc start-build ods-pipeline-manager
oc start-build ods-python-toolset
oc start-build ods-sonar
oc start-build ods-start
.PHONY: start-ods-builds
9 changes: 2 additions & 7 deletions build/package/Dockerfile.gradle-toolset
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,8 @@ COPY build/package/scripts/copy-build-if-cached.sh /usr/local/bin/copy-build-if-
COPY build/package/scripts/copy-artifacts.sh /usr/local/bin/copy-artifacts
COPY build/package/scripts/build-gradle.sh /usr/local/bin/build-gradle
COPY build/package/scripts/supply-sonar-project-properties-default.sh /usr/local/bin/supply-sonar-project-properties-default
COPY build/package/scripts/set-gradle-proxy.sh /usr/local/bin/set-gradle-proxy
RUN chmod +x /usr/local/bin/build-gradle && \
chmod +x /usr/local/bin/cache-build && \
chmod +x /usr/local/bin/copy-build-if-cached && \
chmod +x /usr/local/bin/copy-artifacts && \
chmod +x /usr/local/bin/supply-sonar-project-properties-default && \
chmod +x /usr/local/bin/set-gradle-proxy
COPY build/package/scripts/configure-gradle.sh /usr/local/bin/configure-gradle
COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore

# Add sonar-project.properties
COPY build/package/sonar-project.properties.d/gradle.properties /usr/local/default-sonar-project.properties
Expand Down
5 changes: 4 additions & 1 deletion build/package/Dockerfile.sonar
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@ RUN cd /tmp \
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4

ENV SONAR_EDITION="community" \
OPENSSL_VERSION=1.1
OPENSSL_VERSION=1.1 \
JAVA_HOME=/usr/lib/jvm/jre-11

RUN microdnf install --nodocs openssl-${OPENSSL_VERSION}* java-11-openjdk-headless which && microdnf clean all

COPY --from=builder /usr/local/bin/sonar /usr/local/bin/sonar
COPY --from=builder /usr/local/sonar-scanner-cli /usr/local/sonar-scanner-cli
COPY --from=builder /usr/local/cnes/cnesreport.jar /usr/local/cnes/cnesreport.jar
COPY build/package/scripts/configure-truststore.sh /usr/local/bin/configure-truststore

ENV PATH=/usr/local/sonar-scanner-cli/bin:$PATH

USER 1001
3 changes: 2 additions & 1 deletion build/package/scripts/build-gradle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ echo "Using NEXUS_URL=$NEXUS_URL"
echo "Using GRADLE_OPTS=$GRADLE_OPTS"
echo "Using GRADLE_USER_HOME=$GRADLE_USER_HOME"
echo "Using ARTIFACTS_DIR=$ARTIFACTS_DIR"
mkdir -p "${GRADLE_USER_HOME}"

set-gradle-proxy
configure-gradle

echo
cd "${WORKING_DIR}"
Expand Down
12 changes: 6 additions & 6 deletions build/package/scripts/build-npm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,17 +65,17 @@ if [ "${WORKING_DIR}" != "." ]; then
ARTIFACT_PREFIX="${WORKING_DIR/\//-}-"
fi

echo "Configuring npm to use Nexus ..."
# Remove the protocol segment from NEXUS_URL
NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g')
if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then
echo "Configuring npm to use Nexus (${NEXUS_URL}) ..."
if [ -n "${NEXUS_URL}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then
NEXUS_AUTH="$(urlencode "${NEXUS_USERNAME}"):$(urlencode "${NEXUS_PASSWORD}")"
npm config set registry="$NEXUS_URL"/repository/npmjs/
npm config set always-auth=true
npm config set _auth="$(echo -n "$NEXUS_AUTH" | base64)"
npm config set [email protected]
npm config set ca=null
npm config set strict-ssl=false
if [ -f /etc/ssl/certs/private-cert.pem ]; then
echo "Configuring private cert ..."
npm config set cafile=/etc/ssl/certs/private-cert.pem
fi
fi;

echo "package-*.json checks ..."
Expand Down
2 changes: 1 addition & 1 deletion build/package/scripts/build-python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ if [ "${WORKING_DIR}" != "." ]; then
ARTIFACT_PREFIX="${WORKING_DIR/\//-}-"
fi

echo "Configuring pip to use Nexus ..."
echo "Configuring pip to use Nexus (${NEXUS_URL}) ..."
# Remove the protocol segment from NEXUS_URL
NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g')
if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWORD}" ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,20 @@
# This script checks for env variable HTTP_PROXY and adds them to gradle.properties.
CONTENT=""

if [[ $HTTP_PROXY != "" ]]; then
if [ -f /etc/ssl/certs/private-cert.pem ]; then
echo "Configuring Gradle to trust private cert ..."
configure-truststore --dest-store ".ods-cache/keystore/cacerts" --dest-storepass password
# shellcheck disable=SC2181
if [ $? -ne 0 ]; then
exit 1
fi
# Configure Gradle to use the modified trust store.
CONTENT+="systemProp.javax.net.ssl.trustStore=.ods-cache/keystore/cacerts\n"
CONTENT+="systemProp.javax.net.ssl.trustStorePassword=password\n"
fi

if [ "${HTTP_PROXY}" != "" ]; then
echo "Configuring Gradle to honor HTTP_PROXY ..."
proxy=$(echo "$HTTP_PROXY" | sed -e "s|https://||g" | sed -e "s|http://||g")
proxy_hostp=$(echo "$proxy" | cut -d "@" -f2)

Expand Down Expand Up @@ -32,7 +44,8 @@ if [[ $HTTP_PROXY != "" ]]; then
fi
fi

if [[ $NO_PROXY != "" ]]; then
if [ "${NO_PROXY}" != "" ]; then
echo "Configuring Gradle to honor NO_PROXY ..."
# shellcheck disable=SC2001
noproxy_host=$(echo "$NO_PROXY" | sed -e 's|\,\.|\,\*\.|g')
# shellcheck disable=SC2001
Expand All @@ -41,6 +54,6 @@ if [[ $NO_PROXY != "" ]]; then
CONTENT+="systemProp.https.nonProxyHosts=$noproxy_host\n"
fi

if [[ $CONTENT != "" ]]; then
if [ "${CONTENT}" != "" ]; then
echo -e "$CONTENT" > "${GRADLE_USER_HOME}/gradle.properties"
fi
59 changes: 59 additions & 0 deletions build/package/scripts/configure-truststore.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/bin/bash
set -u

md5_bin="${MD5_BIN:-"md5sum --tag"}"
private_cert="/etc/ssl/certs/private-cert.pem"
src_truststore="${JAVA_HOME}/lib/security/cacerts"
src_pass="changeit"
dest_pass="changeit"

while [[ "$#" -gt 0 ]]; do
case $1 in

--src-store) src_truststore="$2"; shift;;
--src-store=*) src_truststore="${1#*=}";;

--src-storepass) src_pass="$2"; shift;;
--src-storepass=*) src_pass="${1#*=}";;

--dest-store) dest_truststore="$2"; shift;;
--dest-store=*) dest_truststore="${1#*=}";;

--dest-storepass) dest_pass="$2"; shift;;
--dest-storepass=*) dest_pass="${1#*=}";;

--debug) set -x; shift;;

*) echo "Unknown parameter passed: $1"; exit 1;;
esac; shift; done

dest_truststore_dir="${dest_truststore%/*}"
mkdir -p "${dest_truststore_dir}"

md5_private_cert_path="${dest_truststore_dir}/.md5-private-cert"

if [ -f "${private_cert}" ]; then
md5_private_cert=$(${md5_bin} "${private_cert}")
if [ ! -f "${dest_truststore}" ] || [ "${md5_private_cert}" != "$(cat "${md5_private_cert_path}")" ]; then
echo "Creating truststore with private cert ..."
# Copy global keystone to location where we can write to (hide output containing warnings).
keytool -importkeystore \
-srckeystore "${src_truststore}" -destkeystore "${dest_truststore}" \
-deststorepass "${dest_pass}" -srcstorepass "${src_pass}" &> keytool-output.txt
# shellcheck disable=SC2181
if [ $? -ne 0 ]; then
cat keytool-output.txt; exit 1
fi
# Trust private cert (hide output containing warnings).
keytool -importcert -noprompt -trustcacerts \
-alias private-cert -file "${private_cert}" \
-keystore "${dest_truststore}" -storepass "${dest_pass}" &> keytool-output.txt
# shellcheck disable=SC2181
if [ $? -ne 0 ]; then
cat keytool-output.txt; exit 1
fi
echo "${md5_private_cert}" > "${md5_private_cert_path}"
fi
else
echo "No private cert found, skipping ..."
fi
44 changes: 19 additions & 25 deletions build/package/scripts/download-aqua-scanner.sh
Original file line number Diff line number Diff line change
@@ -1,45 +1,39 @@
#!/bin/bash
set -eu

md5bin="${MD5_BIN:-"md5sum --tag"}"
debug="${DEBUG:-false}"
aquaScannerUrl=""
binDir=".ods-cache/bin"
md5_bin="${MD5_BIN:-"md5sum --tag"}"
aqua_scanner_url=""
bin_dir=".ods-cache/bin"

while [[ "$#" -gt 0 ]]; do
case $1 in

--bin-dir) binDir="$2"; shift;;
--bin-dir=*) binDir="${1#*=}";;
--bin-dir) bin_dir="$2"; shift;;
--bin-dir=*) bin_dir="${1#*=}";;

--aqua-scanner-url) aquaScannerUrl="$2"; shift;;
--aqua-scanner-url=*) aquaScannerUrl="${1#*=}";;
--aqua-scanner-url) aqua_scanner_url="$2"; shift;;
--aqua-scanner-url=*) aqua_scanner_url="${1#*=}";;

--debug) debug="$2"; shift;;
--debug=*) debug="${1#*=}";;
--debug) set -x; shift;;

*) echo "Unknown parameter passed: $1"; exit 1;;
esac; shift; done

if [ "${debug}" == "true" ]; then
set -x
fi

aquaScannerPath="${binDir}/aquasec"
md5AquaScannerUrlPath="${binDir}/.md5-aquasec"
aqua_scanner_path="${bin_dir}/aquasec"
md5_aqua_scanner_url_path="${bin_dir}/.md5-aquasec"

# Optionally install Aqua scanner.
# If the binary already exists and was downloaded from the
# URL given by aquaScannerUrl, skip download.
if [ -n "${aquaScannerUrl}" ] && [ "${aquaScannerUrl}" != "none" ]; then
md5AquaScannerUrl=$(${md5bin} -s "${aquaScannerUrl}")
if [ ! -f "${md5AquaScannerUrlPath}" ] || [ "${md5AquaScannerUrl}" != "$(cat "${md5AquaScannerUrlPath}")" ]; then
# URL given by aqua_scanner_url, skip download.
if [ -n "${aqua_scanner_url}" ] && [ "${aqua_scanner_url}" != "none" ]; then
md5_aqua_scanner_url=$(${md5_bin} -s "${aqua_scanner_url}")
if [ ! -f "${md5_aqua_scanner_url_path}" ] || [ "${md5_aqua_scanner_url}" != "$(cat "${md5_aqua_scanner_url_path}")" ]; then
echo 'Installing Aqua scanner...'
curl -v -sSf -L "${aquaScannerUrl}" -o aquasec
mv aquasec "${aquaScannerPath}"
chmod +x "${aquaScannerPath}"
echo "${md5AquaScannerUrl}" > "${md5AquaScannerUrlPath}"
curl -v -sSf -L "${aqua_scanner_url}" -o aquasec
mv aquasec "${aqua_scanner_path}"
chmod +x "${aqua_scanner_path}"
echo "${md5_aqua_scanner_url}" > "${md5_aqua_scanner_url_path}"
echo 'Installed Aqua scanner version:'
"${aquaScannerPath}" version
"${aqua_scanner_path}" version
fi
fi
59 changes: 38 additions & 21 deletions cmd/sonar/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,26 @@ import (
)

type options struct {
sonarAuthToken string
sonarURL string
sonarEdition string
workingDir string
rootPath string
qualityGate bool
debug bool
sonarAuthToken string
sonarURL string
sonarEdition string
workingDir string
rootPath string
qualityGate bool
trustStore string
trustStorePassword string
debug bool
}

var defaultOptions = options{
sonarAuthToken: os.Getenv("SONAR_AUTH_TOKEN"),
sonarURL: os.Getenv("SONAR_URL"),
sonarEdition: os.Getenv("SONAR_EDITION"),
workingDir: ".",
qualityGate: false,
trustStore: "cacerts",
trustStorePassword: "password",
debug: (os.Getenv("DEBUG") == "true"),
}

func main() {
Expand All @@ -31,12 +44,14 @@ func main() {
}

opts := options{rootPath: rootPath}
flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", os.Getenv("SONAR_AUTH_TOKEN"), "sonar-auth-token")
flag.StringVar(&opts.sonarURL, "sonar-url", os.Getenv("SONAR_URL"), "sonar-url")
flag.StringVar(&opts.sonarEdition, "sonar-edition", os.Getenv("SONAR_EDITION"), "sonar-edition")
flag.StringVar(&opts.workingDir, "working-dir", ".", "working directory")
flag.BoolVar(&opts.qualityGate, "quality-gate", false, "require quality gate pass")
flag.BoolVar(&opts.debug, "debug", (os.Getenv("DEBUG") == "true"), "debug mode")
flag.StringVar(&opts.sonarAuthToken, "sonar-auth-token", defaultOptions.sonarAuthToken, "sonar-auth-token")
flag.StringVar(&opts.sonarURL, "sonar-url", defaultOptions.sonarURL, "sonar-url")
flag.StringVar(&opts.sonarEdition, "sonar-edition", defaultOptions.sonarEdition, "sonar-edition")
flag.StringVar(&opts.workingDir, "working-dir", defaultOptions.workingDir, "working directory")
flag.BoolVar(&opts.qualityGate, "quality-gate", defaultOptions.qualityGate, "require quality gate pass")
flag.StringVar(&opts.trustStore, "truststore", defaultOptions.trustStore, "JKS truststore")
flag.StringVar(&opts.trustStorePassword, "truststore-password", defaultOptions.trustStorePassword, "JKS truststore password")
flag.BoolVar(&opts.debug, "debug", defaultOptions.debug, "debug mode")
flag.Parse()

var logger logging.LeveledLoggerInterface
Expand All @@ -58,11 +73,13 @@ func main() {
}

sonarClient, err := sonar.NewClient(&sonar.ClientConfig{
APIToken: opts.sonarAuthToken,
BaseURL: opts.sonarURL,
ServerEdition: opts.sonarEdition,
Debug: opts.debug,
Logger: logger,
APIToken: opts.sonarAuthToken,
BaseURL: opts.sonarURL,
ServerEdition: opts.sonarEdition,
TrustStore: opts.trustStore,
TrustStorePassword: opts.trustStorePassword,
Debug: opts.debug,
Logger: logger,
})
if err != nil {
log.Fatal("sonar client:", err)
Expand Down Expand Up @@ -96,17 +113,17 @@ func sonarScan(
Base: ctxt.PullRequestBase,
}
}
scanStdout, err := sonarClient.Scan(
err := sonarClient.Scan(
sonarProject,
ctxt.GitRef,
ctxt.GitCommitSHA,
prInfo,
os.Stdout,
os.Stdin,
)
if err != nil {
logger.Infof(scanStdout)
return fmt.Errorf("scan failed: %w", err)
}
logger.Infof(scanStdout)

logger.Infof("Wait until compute engine task finishes ...")
err = waitUntilComputeEngineTaskIsSuccessful(logger, sonarClient)
Expand Down
3 changes: 2 additions & 1 deletion cmd/start/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ func cleanCache(fsb FileSystemBase, fnRemove RemoveFunc, expirationDays int) err
cacheDependenciesPath := filepath.Join(".", odsCacheDependenciesDirName)

// To avoid spare files inside the dependency cache which are not supported,
// delete all other areas of the cache.
// delete all unknown folders in there.
// Files outside the dependency cache are left untouched.
dirEntryFunc := func(path string, d fs.DirEntry) WalkAndRemovalFlags {
if !strings.HasPrefix(path, cacheDependenciesPath) {
return 0 // allow files outside the dependency cache area for experimentation
Expand Down
5 changes: 1 addition & 4 deletions deploy/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,15 @@ Manifests in `ods-pipeline` are applied once per project by a project administra

## Subcharts

The `tasks`, `images` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift.
The `tasks` and `setup` subcharts are maintained in https://github.com/opendevstack/ods-pipeline, and may be used by project admins to control the deployment of ODS pipeline resources in the respective project namespace in OpenShift.

### Subcharts Contents

The resources are defined using Helm:
* `BuildConfig` and `ImageStream` resources (in the `images` subchart)
* `Task` resources (in `tasks` subchart)
* `ConfigMap` and `Secret` resources used by ODS tasks (in `setup` subchart)
* ODS pipeline manager (`Service`/`Deployment`) (in `setup` subchart)

The resources of the `images` subchart are only applicable for OpenShift clusters. The subcharts may individually be enabled or disabled via the umbrella chart's `values.yaml`.

### Versioning

In a KinD cluster there are no versions. Images use the implicit `latest` tag. That makes testing and local development easy.
Expand Down
Loading

0 comments on commit fd21339

Please sign in to comment.