From 336d935a7f86fe8ffaf5a379d005aeb92b261f84 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Sat, 28 Oct 2017 09:47:22 +0100 Subject: [PATCH 01/13] Remove Goose and migrate to Stratos DBMigrator --- deploy/build_portal_proxy.sh | 5 ++ deploy/ci/tasks/stratos-ui/run-tests.yml | 2 +- deploy/common.yml | 4 +- deploy/compile.sh | 7 +- deploy/db/Dockerfile.goose.dev | 12 ++-- deploy/db/Dockerfile.k8s.postflight-job | 6 +- deploy/db/scripts/development.sh | 12 ++-- deploy/db/scripts/run-postflight-job.k8s.sh | 10 +-- deploy/docker-compose/build.sh | 3 - deploy/kubernetes/build.sh | 20 +++--- .../Dockerfile.stratos-goose-base.tmpl | 2 - .../stratos-base-images/build-base-images.sh | 16 +---- .../tools/build-postflight-image-builder.sh | 67 ------------------- .../tools/build-push-proxy-builder-image.sh | 4 +- 14 files changed, 48 insertions(+), 122 deletions(-) delete mode 100644 deploy/stratos-base-images/Dockerfile.stratos-goose-base.tmpl delete mode 100755 deploy/tools/build-postflight-image-builder.sh diff --git a/deploy/build_portal_proxy.sh b/deploy/build_portal_proxy.sh index cbc7cd7ed7..2a2b8fb3b8 100755 --- a/deploy/build_portal_proxy.sh +++ b/deploy/build_portal_proxy.sh @@ -6,12 +6,17 @@ set -x pushd $(git rev-parse --show-toplevel) +BUILD_ARGS="" +if [ ! -z "${BUILD_MIGRATOR}" ]; then + BUILD_ARGS='-e BUILD_DB_MIGRATOR="true"' +fi docker run -it \ --rm \ -e USER_NAME=$(id -nu) \ -e USER_ID=$(id -u) \ -e GROUP_ID=$(id -g) \ + ${BUILD_ARGS} \ --name console-proxy-builder \ --volume $(pwd):/go/src/github.com/SUSE/stratos-ui \ splatform/stratos-proxy-builder:opensuse $* diff --git a/deploy/ci/tasks/stratos-ui/run-tests.yml b/deploy/ci/tasks/stratos-ui/run-tests.yml index a29e43e1fe..655eebbabd 100644 --- a/deploy/ci/tasks/stratos-ui/run-tests.yml +++ b/deploy/ci/tasks/stratos-ui/run-tests.yml @@ -55,7 +55,7 @@ run: cat ../build/secrets.json - for image in splatform/stratos-goose-base:opensuse splatform/stratos-bk-base:opensuse splatform/stratos-nginx-base:opensuse splatform/stratos-uaa splatform/stratos-concourse; do + for image in splatform/stratos-bk-base:opensuse splatform/stratos-nginx-base:opensuse splatform/stratos-uaa splatform/stratos-concourse; do docker pull ${REGISTRY_NAME}/$image docker tag ${REGISTRY_NAME}/$image $image done diff --git a/deploy/common.yml b/deploy/common.yml index 836e581d56..d9dc7d5d73 100644 --- a/deploy/common.yml +++ b/deploy/common.yml @@ -36,8 +36,8 @@ services: goose: extends: buildCommon build: - context: ../deploy - dockerfile: db/Dockerfile.goose.dev + context: ../ + dockerfile: deploy/db/Dockerfile.goose.dev environment: - DATABASE_PROVIDER=mysql diff --git a/deploy/compile.sh b/deploy/compile.sh index cfd8e5bf4c..fbff63bfe5 100755 --- a/deploy/compile.sh +++ b/deploy/compile.sh @@ -2,9 +2,12 @@ set -e rm -rf node_modules -npm install +npm install --production +if [ ! -z ${BUILD_DB_MIGRATOR} ]; then +npm run build-migrator; +else npm run build-backend - +fi if [ "${USER_NAME}" != "root" ]; then useradd -G users -u ${USER_ID} ${USER_NAME} chown -R ${USER_NAME}:${GROUP_ID} outputs/ diff --git a/deploy/db/Dockerfile.goose.dev b/deploy/db/Dockerfile.goose.dev index 538562719d..a707100904 100644 --- a/deploy/db/Dockerfile.goose.dev +++ b/deploy/db/Dockerfile.goose.dev @@ -1,11 +1,13 @@ -FROM splatform/stratos-goose-base:opensuse +FROM splatform/stratos-bk-base:opensuse WORKDIR /src -COPY db/dbconf.yml db/dbconf.yml -COPY db/migrations db/migrations -COPY db/scripts/development.sh . -COPY db/scripts/wait-for-it.sh . +COPY deploy/db/dbconf.yml db/dbconf.yml +COPY deploy/db/migrations db/migrations +COPY deploy/db/scripts/development.sh . +COPY deploy/db/scripts/wait-for-it.sh . +COPY outputs/stratos-dbmigrator . +RUN chmod +x stratos-dbmigrator RUN chmod +x wait-for-it.sh RUN chmod +x development.sh diff --git a/deploy/db/Dockerfile.k8s.postflight-job b/deploy/db/Dockerfile.k8s.postflight-job index 7bb03d8f0c..0544ed3152 100644 --- a/deploy/db/Dockerfile.k8s.postflight-job +++ b/deploy/db/Dockerfile.k8s.postflight-job @@ -1,8 +1,8 @@ -FROM splatform/stratos-goose-base:opensuse +FROM splatform/stratos-bk-base:opensuse RUN zypper -n in postgresql mariadb-client -RUN go get github.com/go-sql-driver/mysql -COPY goose /usr/local/bin/ +COPY outputs/stratos-dbmigrator /usr/local/bin/ +RUN chmod +x /usr/local/bin/stratos-dbmigrator COPY deploy/db/dbconf.yml db/dbconf.yml COPY deploy/db/migrations db/migrations COPY deploy/db/scripts/run-postflight-job.k8s.sh /run-postflight-job.sh diff --git a/deploy/db/scripts/development.sh b/deploy/db/scripts/development.sh index 8e6bd21313..9e007a9c14 100755 --- a/deploy/db/scripts/development.sh +++ b/deploy/db/scripts/development.sh @@ -3,32 +3,32 @@ set -e # Check the version echo "Checking database version." -$GOPATH/bin/goose --env=mariadb-development dbversion +./stratos-dbmigrator --env=mariadb-development dbversion EXITVAL=$? while [ $EXITVAL -ne 0 ] do echo "Failed to execute dbversion check, retrying after one second" sleep 1 - $GOPATH/bin/goose --env=mariadb-development dbversion + ./stratos-dbmigrator --env=mariadb-development dbversion EXITVAL=$? done # Check the status echo "Checking database status." -$GOPATH/bin/goose --env=mariadb-development status +./stratos-dbmigrator --env=mariadb-development status # Run migrations echo "Attempting database migrations." -$GOPATH/bin/goose --env=mariadb-development up +./stratos-dbmigrator --env=mariadb-development up # CHeck the status echo "Checking database status." -$GOPATH/bin/goose --env=mariadb-development status +./stratos-dbmigrator --env=mariadb-development status # Check the version echo "Checking database version." -$GOPATH/bin/goose --env=mariadb-development dbversion +./stratos-dbmigrator --env=mariadb-development dbversion echo "Database operation(s) complete." diff --git a/deploy/db/scripts/run-postflight-job.k8s.sh b/deploy/db/scripts/run-postflight-job.k8s.sh index 383f7c32e2..6a667d658f 100755 --- a/deploy/db/scripts/run-postflight-job.k8s.sh +++ b/deploy/db/scripts/run-postflight-job.k8s.sh @@ -57,23 +57,23 @@ echo "DBCONFIG: $DBCONF_KEY" echo "Connection string: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true" # Check the version echo "Checking database version." -goose --env=$DBCONF_KEY dbversion +stratos-dbmigrator --env=$DBCONF_KEY dbversion # Check the status echo "Checking database status." -goose --env=$DBCONF_KEY status +stratos-dbmigrator --env=$DBCONF_KEY status # Run migrations echo "Attempting database migrations." -goose --env=$DBCONF_KEY up +stratos-dbmigrator --env=$DBCONF_KEY up # CHeck the status echo "Checking database status." -goose --env=$DBCONF_KEY status +stratos-dbmigrator --env=$DBCONF_KEY status # Check the version echo "Checking database version." -goose --env=$DBCONF_KEY dbversion +stratos-dbmigrator --env=$DBCONF_KEY dbversion echo "Database operation(s) complete." diff --git a/deploy/docker-compose/build.sh b/deploy/docker-compose/build.sh index 1e2f2be87f..4bade6ba07 100755 --- a/deploy/docker-compose/build.sh +++ b/deploy/docker-compose/build.sh @@ -123,8 +123,6 @@ function cleanup { echo "-- Cleaning up ${STRATOS_UI_PATH}/deploy/containers/nginx/dist" rm -rf ${STRATOS_UI_PATH}/deploy/containers/nginx/dist - rm -f ${STRATOS_UI_PATH}/goose - } function updateTagForRelease { @@ -192,7 +190,6 @@ function buildGoose { # Build the postflight container echo echo "-- Build & publish the runtime container image for the postflight job" - preloadImage splatform/stratos-goose-base:opensuse buildAndPublishImage stratos-dc-goose ./db/Dockerfile.goose.dev ${STRATOS_UI_PATH}/deploy rm -f ${STRATOS_UI_PATH}/goose } diff --git a/deploy/kubernetes/build.sh b/deploy/kubernetes/build.sh index b0de922c97..150b30e1a4 100755 --- a/deploy/kubernetes/build.sh +++ b/deploy/kubernetes/build.sh @@ -131,8 +131,6 @@ function cleanup { echo "-- Cleaning up ${STRATOS_UI_PATH}/deploy/containers/nginx/dist" rm -rf ${STRATOS_UI_PATH}/deploy/containers/nginx/dist - rm -f ${STRATOS_UI_PATH}/goose - } function preloadImage { @@ -241,16 +239,20 @@ function buildPostflightJob { # Build the postflight container echo echo "-- Build & publish the runtime container image for the postflight job" - docker run \ - ${RUN_ARGS} \ + pushd ${STRATOS_UI_PATH} > /dev/null 2>&1 + docker run ${RUN_ARGS} \ -it \ --rm \ - --name postflight-builder \ - --volume $(pwd):/go/bin/ \ - ${DOCKER_REGISTRY}/${DOCKER_ORG}/stratos-postflight-builder:${BASE_IMAGE_TAG} - mv goose ${STRATOS_UI_PATH}/ + -e USER_NAME=$(id -nu) \ + -e USER_ID=$(id -u) \ + -e GROUP_ID=$(id -g) \ + -e BUILD_DB_MIGRATOR="true" \ + --name stratos-proxy-builder \ + --volume $(pwd):/go/src/github.com/SUSE/stratos-ui \ + ${DOCKER_REGISTRY}/${DOCKER_ORG}/stratos-proxy-builder:${BASE_IMAGE_TAG} buildAndPublishImage stratos-postflight-job deploy/db/Dockerfile.k8s.postflight-job ${STRATOS_UI_PATH} - rm -f ${STRATOS_UI_PATH}/goose + popd > /dev/null 2>&1 + } function buildUI { diff --git a/deploy/stratos-base-images/Dockerfile.stratos-goose-base.tmpl b/deploy/stratos-base-images/Dockerfile.stratos-goose-base.tmpl deleted file mode 100644 index 079a93b3d8..0000000000 --- a/deploy/stratos-base-images/Dockerfile.stratos-goose-base.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -FROM {{GO_BUILD_BASE}} -RUN go get bitbucket.org/liamstask/goose/cmd/goose \ No newline at end of file diff --git a/deploy/stratos-base-images/build-base-images.sh b/deploy/stratos-base-images/build-base-images.sh index 415f3c72d1..d857c1add8 100755 --- a/deploy/stratos-base-images/build-base-images.sh +++ b/deploy/stratos-base-images/build-base-images.sh @@ -97,10 +97,6 @@ build_bk_build_base(){ build_and_push_image stratos-bk-build-base Dockerfile.stratos-bk-build-base } -build_goose_base(){ - build_and_push_image stratos-goose-base Dockerfile.stratos-goose-base -} - build_portal_proxy_builder(){ pushd ${DEPLOY_PATH}/ BK_BUILD_BASE=${REGISTRY}/${ORGANIZATION}/stratos-bk-build-base:${TAG} @@ -108,12 +104,6 @@ build_portal_proxy_builder(){ popd } -build_postflight_job_base(){ - pushd ${DEPLOY_PATH}/ - TAG=${TAG} DOCKER_REGISTRY=${REGISTRY} DOCKER_ORG=${ORGANIZATION} tools/build-postflight-image-builder.sh - popd -} - build_preflight_job_base(){ build_and_push_image stratos-preflight-base Dockerfile.stratos-preflight-base } @@ -124,8 +114,6 @@ build_go_base build_ui_base; # Used for running the backend build_bk_base; -# Used for goose -build_goose_base; # Used for hosting nginx build_nginx_base; # Used for stratos-proxy-builder base @@ -133,7 +121,5 @@ build_bk_build_base; # Used for building the backend build_portal_proxy_builder; # Used for building the postflight job image -build_postflight_job_base; -# Used for building the preflight job image build_preflight_job_base; -rm -f mo; \ No newline at end of file +rm -f mo; diff --git a/deploy/tools/build-postflight-image-builder.sh b/deploy/tools/build-postflight-image-builder.sh deleted file mode 100755 index 13187c08bc..0000000000 --- a/deploy/tools/build-postflight-image-builder.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -DIR_NAME=$(mktemp -d) -DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} -DOCKER_ORGANISATION=${DOCKER_ORG:-splatform} -IMAGE_NAME=stratos-postflight-builder -TAG=${TAG:-test} - -while getopts ":ho:t:p" opt ; do - case $opt in - o) - DOCKER_ORGANISATION=true - ;; - t) - TAG=true - ;; - p) - PUSH_IMAGE=true - ;; - h) - usage - ;; - \?) - echo "Invalid option -$OPTARG" >&2 - usage - ;; - esac -done - -PROG=$(basename ${BASH_SOURCE[0]}) - -function usage { - echo "usage: $PROG [-o] [-t] [-p]" - echo " -o Organisation to push image to" - echo " -t Tag of the image" - echo " -p Push image" - exit 1 -} - -# Write out script to build goose - -ENTRYPOINT_SCRIPT=${DIR_NAME}/build-goose.sh -cat << EOT >> ${ENTRYPOINT_SCRIPT} -#!/usr/bin/env sh - -go get 'bitbucket.org/liamstask/goose/cmd/goose' -EOT - -chmod +x ${DIR_NAME}/build-goose.sh - -# Write out Dockerfile -DOCKERFILE=${DIR_NAME}/Dockerfile -cat << EOT >> ${DOCKERFILE} -FROM splatform/stratos-go-build-base:opensuse - -ADD build-goose.sh /build-goose.sh -CMD ["/build-goose.sh"] -EOT - -cd ${DIR_NAME} -echo "Building image ${DOCKER_ORGANISATION}/${IMAGE_NAME}:${TAG}" -docker build . -t ${DOCKER_REGISTRY}/${DOCKER_ORGANISATION}/${IMAGE_NAME}:${TAG} - -if [ -n "${PUSH_IMAGE}" ]; then - echo "Pushing image ${DOCKER_ORGANISATION}/${IMAGE_NAME}:${TAG}" - docker push ${DOCKER_REGISTRY}/${DOCKER_ORGANISATION}/${IMAGE_NAME}:${TAG} -fi \ No newline at end of file diff --git a/deploy/tools/build-push-proxy-builder-image.sh b/deploy/tools/build-push-proxy-builder-image.sh index 394bfd4286..e26263eef5 100755 --- a/deploy/tools/build-push-proxy-builder-image.sh +++ b/deploy/tools/build-push-proxy-builder-image.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -eux -DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} +DOCKER_REGISTRY=${DOCKER_REGISTRY:-registry.capbristol.com:5000} DOCKER_ORG=${DOCKER_ORG:-splatform} NAME=stratos-proxy-builder -TAG=${TAG:-test} +TAG=${TAG:-opensuse} BK_BUILD_BASE=${BK_BUILD_BASE:-splatform/stratos-bk-build-base:opensuse} STRATOS_UI_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../../" From e011fd30a1684001b76fa29fb3c0bc08b402b677 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Sat, 28 Oct 2017 09:59:23 +0100 Subject: [PATCH 02/13] Update build-push-proxy-builder-image.sh --- deploy/tools/build-push-proxy-builder-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/tools/build-push-proxy-builder-image.sh b/deploy/tools/build-push-proxy-builder-image.sh index e26263eef5..fc609b36f3 100755 --- a/deploy/tools/build-push-proxy-builder-image.sh +++ b/deploy/tools/build-push-proxy-builder-image.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eux -DOCKER_REGISTRY=${DOCKER_REGISTRY:-registry.capbristol.com:5000} +DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} DOCKER_ORG=${DOCKER_ORG:-splatform} NAME=stratos-proxy-builder TAG=${TAG:-opensuse} From 525247d538df28ae32a29c9a1ded53cf147ad119 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Mon, 30 Oct 2017 09:09:09 +0000 Subject: [PATCH 03/13] Update run-postflight-job.k8s.sh --- deploy/db/scripts/run-postflight-job.k8s.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/db/scripts/run-postflight-job.k8s.sh b/deploy/db/scripts/run-postflight-job.k8s.sh index 6a667d658f..3181066a5d 100755 --- a/deploy/db/scripts/run-postflight-job.k8s.sh +++ b/deploy/db/scripts/run-postflight-job.k8s.sh @@ -5,7 +5,7 @@ function execStatement { stmt=$1 if [ "$DATABASE_PROVIDER" = "mysql" ]; then - echo "Executing: mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt" + echo "Executing: mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p****** -e $stmt" mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt fi @@ -54,7 +54,7 @@ fi echo "Checking database to see if migration is necessary." echo "DBCONFIG: $DBCONF_KEY" -echo "Connection string: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true" +echo "Connection string: $DB_USER:********@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true" # Check the version echo "Checking database version." stratos-dbmigrator --env=$DBCONF_KEY dbversion From ac7b1f14c02e3dc41cc8548be0f0b09efa8bf7b3 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Mon, 30 Oct 2017 12:41:03 +0000 Subject: [PATCH 04/13] wip --- deploy/common.yml | 6 ++- deploy/db/Dockerfile.mariadb | 9 ++++ deploy/db/mariadb-entrypoint.sh | 47 +++++++++++++++++++ .../Dockerfile.stratos-mariadb-base.tmpl | 17 +++++++ .../stratos-base-images/build-base-images.sh | 6 +++ 5 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 deploy/db/Dockerfile.mariadb create mode 100755 deploy/db/mariadb-entrypoint.sh create mode 100644 deploy/stratos-base-images/Dockerfile.stratos-mariadb-base.tmpl diff --git a/deploy/common.yml b/deploy/common.yml index 836e581d56..a81d813d2b 100644 --- a/deploy/common.yml +++ b/deploy/common.yml @@ -25,8 +25,10 @@ services: mariadb: - extends: imageCommon - image: mariadb:latest + extends: buildCommon + build: + context: ../deploy/db + dockerfile: Dockerfile.mariadb env_file: ./mariadb.env ports: diff --git a/deploy/db/Dockerfile.mariadb b/deploy/db/Dockerfile.mariadb new file mode 100644 index 0000000000..948eb9ff66 --- /dev/null +++ b/deploy/db/Dockerfile.mariadb @@ -0,0 +1,9 @@ +FROM splatform/stratos-db-base:opensuse + +COPY mariadb-entrypoint.sh /docker-entrypoint.sh + +# ENTRYPOINT +ENTRYPOINT ["/docker-entrypoint.sh"] + +EXPOSE 3306 +CMD ["mysqld_safe"] diff --git a/deploy/db/mariadb-entrypoint.sh b/deploy/db/mariadb-entrypoint.sh new file mode 100755 index 0000000000..5401a19239 --- /dev/null +++ b/deploy/db/mariadb-entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +MYSQL_DATADIR="/var/lib/mysql" + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + # if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" ]; then + # echo >&2 'error: database is uninitialized and MYSQL_ROOT_PASSWORD not set' + # echo >&2 ' Did you forget to add -e MYSQL_ROOT_PASSWORD=... ?' + # exit 1 + # fi + + echo 'Running mysql_install_db ...' + mysql_install_db --datadir="$MYSQL_DATADIR" + echo 'Finished mysql_install_db' + + # These statements _must_ be on individual lines, and _must_ end with + # semicolons (no line breaks or comments are permitted). + # TODO proper SQL escaping on ALL the things D: + + tempSqlFile='/tmp/mysql-first-time.sql' + cat > "$tempSqlFile" <<-EOSQL +DELETE FROM mysql.user ; +CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; +GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; +DROP DATABASE IF EXISTS test ; +EOSQL + + if [ "$MYSQL_DATABASE" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" + fi + + if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + + if [ "$MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi + fi + + echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" + set -- "$@" --init-file="$tempSqlFile" +fi + +chown -R mysql:mysql "$MYSQL_DATADIR" + +exec "$@" diff --git a/deploy/stratos-base-images/Dockerfile.stratos-mariadb-base.tmpl b/deploy/stratos-base-images/Dockerfile.stratos-mariadb-base.tmpl new file mode 100644 index 0000000000..9c71d9e705 --- /dev/null +++ b/deploy/stratos-base-images/Dockerfile.stratos-mariadb-base.tmpl @@ -0,0 +1,17 @@ +FROM opensuse:latest + +# Default password for root of mariadb +ENV MYSQL_ROOT_PASSWORD mysecretpassword + +# Add repo for the latest mariadb +RUN zypper in -y mariadb net-tools \ + && zypper clean --all + +# Config for mariadb +RUN rm -rf /var/lib/mysql \ + && mkdir -p /var/lib/mysql \ + && mkdir -p /var/log/mysql \ + && touch /var/log/mysql/mysqld.log \ + && chown -R mysql:mysql /var/log/mysql + +VOLUME ["/var/lib/mysql"] \ No newline at end of file diff --git a/deploy/stratos-base-images/build-base-images.sh b/deploy/stratos-base-images/build-base-images.sh index 415f3c72d1..3326256c4b 100755 --- a/deploy/stratos-base-images/build-base-images.sh +++ b/deploy/stratos-base-images/build-base-images.sh @@ -118,6 +118,10 @@ build_preflight_job_base(){ build_and_push_image stratos-preflight-base Dockerfile.stratos-preflight-base } +build_mariadb_base(){ + build_and_push_image stratos-db-base Dockerfile.stratos-mariadb-base +} + # Base with go build_go_base # Used building the UI @@ -136,4 +140,6 @@ build_portal_proxy_builder; build_postflight_job_base; # Used for building the preflight job image build_preflight_job_base; +# Used for building the DB image +build_mariadb_base; rm -f mo; \ No newline at end of file From 51bf55a36721d19221563fd12c5485987129030e Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Mon, 30 Oct 2017 15:11:05 +0000 Subject: [PATCH 05/13] Migrate helm chart to use custom mariadb image --- deploy/kubernetes/build.sh | 8 ++ deploy/kubernetes/console/requirements.lock | 6 -- deploy/kubernetes/console/requirements.yaml | 4 - .../console/templates/deployment.yaml | 83 ++++++++++++++++--- .../console/templates/pre-install.yaml | 22 +++++ .../kubernetes/console/templates/secrets.yml | 32 +++++++ .../kubernetes/console/templates/service.yaml | 15 ++++ deploy/kubernetes/console/values.yaml | 11 ++- 8 files changed, 155 insertions(+), 26 deletions(-) delete mode 100644 deploy/kubernetes/console/requirements.lock delete mode 100644 deploy/kubernetes/console/requirements.yaml create mode 100644 deploy/kubernetes/console/templates/secrets.yml diff --git a/deploy/kubernetes/build.sh b/deploy/kubernetes/build.sh index b0de922c97..9ec974d0fc 100755 --- a/deploy/kubernetes/build.sh +++ b/deploy/kubernetes/build.sh @@ -253,6 +253,13 @@ function buildPostflightJob { rm -f ${STRATOS_UI_PATH}/goose } +function buildMariaDb { + echo + echo "-- Building/publishing MariaDB" + # Download and retag image to save bandwidth + buildAndPublishImage stratos-mariadb Dockerfile.mariadb ${STRATOS_UI_PATH}/deploy/db +} + function buildUI { # Prepare the nginx server CURRENT_USER=$ @@ -296,6 +303,7 @@ updateTagForRelease buildProxy buildPreflightJob buildPostflightJob +buildMariaDb buildUI if [ ${CONCOURSE_BUILD:-"not-set"} == "not-set" ]; then diff --git a/deploy/kubernetes/console/requirements.lock b/deploy/kubernetes/console/requirements.lock deleted file mode 100644 index 3c15589e47..0000000000 --- a/deploy/kubernetes/console/requirements.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: mariadb - repository: https://kubernetes-charts.storage.googleapis.com/ - version: 1.0.1 -digest: sha256:ccfff7c376d31903c1df1cdbedde7f46007ea3c43ebffc93f7f922fa0c0ca150 -generated: 2017-10-03T09:42:27.589253916+01:00 diff --git a/deploy/kubernetes/console/requirements.yaml b/deploy/kubernetes/console/requirements.yaml deleted file mode 100644 index 05e5af6f05..0000000000 --- a/deploy/kubernetes/console/requirements.yaml +++ /dev/null @@ -1,4 +0,0 @@ -dependencies: -- name: mariadb - version: 1.0.1 - repository: https://kubernetes-charts.storage.googleapis.com/ diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index 351bc1eb73..69f0f13604 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -1,15 +1,4 @@ --- -apiVersion: v1 -kind: Secret -type: Opaque -metadata: - name: "{{ .Release.Name }}-secret" - labels: - app: "{{ .Release.Name }}" -data: - stolon: {{ .Values.dbPassword | b64enc }} - db-password: {{ .Values.mariadb.mariadbPassword | b64enc }} ---- apiVersion: apps/v1beta1 kind: StatefulSet metadata: @@ -45,7 +34,7 @@ spec: protocol: TCP {{- if .Values.noShared }} - image: {{.Values.dockerRegistry}}/{{.Values.dockerOrg}}/{{.Values.images.postflight}}:{{.Values.consoleVersion}} - name: "{{ .Release.Name }}-postflight" + name: "postflight" env: - name: DB_HOST value: "{{ .Release.Name }}-mariadb" @@ -218,4 +207,72 @@ spec: claimName: "{{ .Release.Name }}-encryption-key-volume" - name: "{{ .Release.Name }}-secret" secret: - secretName: "{{ .Release.Name }}-secret" \ No newline at end of file + secretName: "{{ .Release.Name }}-secret" +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: console-mariadb + labels: + app: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app: "{{ .Release.Name }}" + spec: + containers: + - name: mariadb + image: {{.Values.dockerRegistry}}/{{.Values.dockerOrg}}/{{.Values.images.mariadb}}:{{.Values.consoleVersion}} + env: + {{- if .Values.mariadb.usePassword }} + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-mariadb-secret" + key: mariadb-root-password + {{- if .Values.mariadb.mariadbUser }} + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-mariadb-secret" + key: mariadb-password + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + {{- end }} + - name: MYSQL_USER + value: {{ default "" .Values.mariadb.mariadbUser | quote }} + - name: MYSQL_DATABASE + value: {{ default "" .Values.mariadb.mariadbDatabase | quote }} + ports: + - name: mysql + containerPort: 3306 + livenessProbe: + exec: + command: + - mysqladmin + - ping + initialDelaySeconds: 30 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - mysqladmin + - ping + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: +{{ toYaml .Values.mariadb.resources | indent 10 }} + volumeMounts: + - name: data + mountPath: /var/lib/mysql + volumes: + - name: data + {{- if .Values.mariadb.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.mariadb.persistence.existingClaim | default "console-mariadb" }} + {{- else }} + emptyDir: {} + {{- end -}} diff --git a/deploy/kubernetes/console/templates/pre-install.yaml b/deploy/kubernetes/console/templates/pre-install.yaml index 0d6e4a446e..9a619285aa 100644 --- a/deploy/kubernetes/console/templates/pre-install.yaml +++ b/deploy/kubernetes/console/templates/pre-install.yaml @@ -24,6 +24,28 @@ spec: requests: storage: 1Mi --- +{{- if and .Values.mariadb.persistence.enabled (not .Values.mariadb.persistence.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: console-mariadb + labels: + app: "{{ .Release.Name }}" +spec: + accessModes: + - {{ .Values.mariadb.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.mariadb.persistence.size | quote }} +{{- if .Values.mariadb.persistence.storageClass }} +{{- if (eq "-" .Values.mariadb.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.mariadb.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} +--- apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/deploy/kubernetes/console/templates/secrets.yml b/deploy/kubernetes/console/templates/secrets.yml new file mode 100644 index 0000000000..ec773654d8 --- /dev/null +++ b/deploy/kubernetes/console/templates/secrets.yml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: "{{ .Release.Name }}-secret" + labels: + app: "{{ .Release.Name }}" +data: + stolon: {{ .Values.dbPassword | b64enc }} + db-password: {{ .Values.mariadb.mariadbPassword | b64enc }} +--- +{{- if .Values.mariadb.usePassword -}} +apiVersion: v1 +kind: Secret +metadata: + name: "{{ .Release.Name }}-mariadb-secret" + labels: + app: "{{ .Release.Name }}" +type: Opaque +data: + {{- if .Values.mariadb.mariadbRootPassword }} + mariadb-root-password: {{ .Values.mariadb.mariadbRootPassword | b64enc | quote }} + {{- else }} + mariadb-root-password: {{ randAlphaNum 10 | b64enc | quote }} + {{- end }} + {{- if .Values.mariadb.mariadbPassword }} + mariadb-password: {{ .Values.mariadb.mariadbPassword | b64enc | quote }} + {{- else }} + mariadb-password: {{ randAlphaNum 10 | b64enc | quote }} + {{- end }} +{{- end}} \ No newline at end of file diff --git a/deploy/kubernetes/console/templates/service.yaml b/deploy/kubernetes/console/templates/service.yaml index a7a73e2f08..4428a170c5 100644 --- a/deploy/kubernetes/console/templates/service.yaml +++ b/deploy/kubernetes/console/templates/service.yaml @@ -28,3 +28,18 @@ spec: externalIPs: - {{ .Values.console.externalIP }} {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ .Release.Name }}-mariadb" + labels: + app: "{{ .Release.Name }}" +spec: + type: ClusterIP + ports: + - name: mysql + port: 3306 + targetPort: mysql + selector: + app: "{{ .Release.Name }}" diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index 0a8b115cb3..7d67f89499 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -25,6 +25,8 @@ images: postgres: stratos-postgres preflight: stratos-preflight-job postflight: stratos-postflight-job + mariadb: stratos-mariadb + # Specify which storage class should be used for PVCs #storageClass: default #consoleCert: | @@ -41,18 +43,21 @@ images: mariadb: # Only required for creating the databases mariadbRootPassword: changeme - image: splatform/stratos-mariadb:10.1.28-r2 adminUser: root # Credentials for user mariadbUser: console mariadbPassword: changeme mariadbDatabase: console + usePassword: true + resources: + requests: + memory: 256Mi + cpu: 250m persistence: + enabled: true accessMode: ReadWriteOnce size: 1Gi # storageClass: default - metrics: - enabled: false uaa: protocol: https:// port: From 7db1e78ab3493c8b0b7c539a7a76133de7351084 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Mon, 30 Oct 2017 15:20:53 +0000 Subject: [PATCH 06/13] Remove `dep build` stage --- deploy/ci/tasks/release/create-chart.yml | 1 - deploy/kubernetes/README.md | 5 ----- 2 files changed, 6 deletions(-) diff --git a/deploy/ci/tasks/release/create-chart.yml b/deploy/ci/tasks/release/create-chart.yml index a490c7b85d..128838db82 100644 --- a/deploy/ci/tasks/release/create-chart.yml +++ b/deploy/ci/tasks/release/create-chart.yml @@ -32,7 +32,6 @@ run: cp -f ${ROOT_DIR}/helm-chart-Chart/Chart.yaml* console/Chart.yaml # Generate Helm package - helm dep build console helm package console cp console*.tgz ${ROOT_DIR}/helm-chart/console-helm-chart-${SHORT_GIT_TAG}.tgz cd ${ROOT_DIR}/helm-chart/ diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md index 670bad2609..6c0d3bcdc0 100644 --- a/deploy/kubernetes/README.md +++ b/deploy/kubernetes/README.md @@ -89,11 +89,6 @@ Open a terminal and cd to the `deploy/kubernetes` directory: $ cd deploy/kubernetes ``` -Fetch dependant helm charts -``` -$ helm dep build -``` - Run helm install: ``` From 349fc3ad799e99dfca91b860a467d50201cf11f8 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Mon, 30 Oct 2017 16:26:28 +0000 Subject: [PATCH 07/13] Update create-chart.yml --- deploy/ci/tasks/release/create-chart.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/ci/tasks/release/create-chart.yml b/deploy/ci/tasks/release/create-chart.yml index 128838db82..c5574a5f02 100644 --- a/deploy/ci/tasks/release/create-chart.yml +++ b/deploy/ci/tasks/release/create-chart.yml @@ -19,7 +19,6 @@ run: args: - -exc - | - # Expect this command to fail since k8s isn't available but it will initialise helm locally helm init || true ROOT_DIR=$PWD From 5fda4e274bb574df79cea6a551c446a9acff5055 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Mon, 30 Oct 2017 17:17:33 +0000 Subject: [PATCH 08/13] Tweaks! (#1340) --- deploy/kubernetes/console/Chart.yaml | 2 +- deploy/kubernetes/console/templates/deployment.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/kubernetes/console/Chart.yaml b/deploy/kubernetes/console/Chart.yaml index c8e63f7fec..c608da960d 100644 --- a/deploy/kubernetes/console/Chart.yaml +++ b/deploy/kubernetes/console/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 -description: A Helm chart for deploying Console +description: A Helm chart for deploying Stratos UI Console name: console version: 0.1.0 diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index 351bc1eb73..6b74971830 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -179,8 +179,10 @@ spec: value: https://scf.{{.Values.env.UAA_HOST}}:{{.Values.env.UAA_PORT}} - name: CONSOLE_CLIENT value: cf - - name: CONSOLE_CLIENT_SECRET - value: {{.Values.env.UAA_ADMIN_CLIENT_SECRET}} + - name: CONSOLE_CLIENT_SECRET + value: + - name: AUTO_REG_CF_URL + value: https://api.{{.Values.env.DOMAIN}} - name: CONSOLE_ADMIN_SCOPE value: cloud_controller.admin - name: SKIP_SSL_VALIDATION From 817cc0a0ba0f59246db4a6cdc041fdfa9ebaa609 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Fri, 20 Oct 2017 16:22:32 +0100 Subject: [PATCH 09/13] Add UAA config --- deploy/ci/tasks/stratos-ui/run-tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deploy/ci/tasks/stratos-ui/run-tests.yml b/deploy/ci/tasks/stratos-ui/run-tests.yml index a29e43e1fe..54f5dd688d 100644 --- a/deploy/ci/tasks/stratos-ui/run-tests.yml +++ b/deploy/ci/tasks/stratos-ui/run-tests.yml @@ -47,7 +47,13 @@ run: "user": { "username": "${CONSOLE_USER_USER}", "password": "${CONSOLE_USER_PASSWORD}" - } + }, + "uaa": { + "url": "http://uaa:8080", + "clientId": "console", + "adminUsername": "admin", + "adminPassword": "hscadmin" + }, }, "githubPat": "${GITHUB_TOKEN}" } From d17f10e16692a4ad13da674a94c834f4ead6618a Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Fri, 20 Oct 2017 16:52:03 +0100 Subject: [PATCH 10/13] cherry picked fix --- deploy/ci/tasks/stratos-ui/run-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/ci/tasks/stratos-ui/run-tests.yml b/deploy/ci/tasks/stratos-ui/run-tests.yml index 54f5dd688d..883243f180 100644 --- a/deploy/ci/tasks/stratos-ui/run-tests.yml +++ b/deploy/ci/tasks/stratos-ui/run-tests.yml @@ -47,14 +47,14 @@ run: "user": { "username": "${CONSOLE_USER_USER}", "password": "${CONSOLE_USER_PASSWORD}" - }, - "uaa": { + } + }, + "uaa": { "url": "http://uaa:8080", "clientId": "console", "adminUsername": "admin", "adminPassword": "hscadmin" }, - }, "githubPat": "${GITHUB_TOKEN}" } EOF From 2c3ee9932f18ad8809f5b88674300b9e6ce0e2eb Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Mon, 30 Oct 2017 20:57:35 +0000 Subject: [PATCH 11/13] Removed spurious semicolon --- deploy/compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/compile.sh b/deploy/compile.sh index fbff63bfe5..af68c286b0 100755 --- a/deploy/compile.sh +++ b/deploy/compile.sh @@ -4,7 +4,7 @@ set -e rm -rf node_modules npm install --production if [ ! -z ${BUILD_DB_MIGRATOR} ]; then -npm run build-migrator; +npm run build-migrator else npm run build-backend fi From 709db7d70a97f808816b9219ce0de66b3b0bcf1c Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Tue, 31 Oct 2017 09:44:18 +0000 Subject: [PATCH 12/13] Use scf-values config storage class name if available --- .../console/templates/pre-install.yaml | 19 ++++++++++++------- deploy/kubernetes/console/values.yaml | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/deploy/kubernetes/console/templates/pre-install.yaml b/deploy/kubernetes/console/templates/pre-install.yaml index 9a619285aa..9ffb9206b4 100644 --- a/deploy/kubernetes/console/templates/pre-install.yaml +++ b/deploy/kubernetes/console/templates/pre-install.yaml @@ -8,6 +8,8 @@ metadata: annotations: {{- if .Values.storageClass }} volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} + {{- else if .Values.kube.storage_class.persistent }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} {{- else }} volume.alpha.kubernetes.io/storage-class: default {{- end }} @@ -31,19 +33,20 @@ metadata: name: console-mariadb labels: app: "{{ .Release.Name }}" + annotations: + {{- if .Values.storageClass }} + volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} + {{- else if .Values.kube.storage_class.persistent }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} + {{- else }} + volume.alpha.kubernetes.io/storage-class: default + {{- end }} spec: accessModes: - {{ .Values.mariadb.persistence.accessMode | quote }} resources: requests: storage: {{ .Values.mariadb.persistence.size | quote }} -{{- if .Values.mariadb.persistence.storageClass }} -{{- if (eq "-" .Values.mariadb.persistence.storageClass) }} - storageClassName: "" -{{- else }} - storageClassName: "{{ .Values.mariadb.persistence.storageClass }}" -{{- end }} -{{- end }} {{- end }} --- apiVersion: v1 @@ -55,6 +58,8 @@ metadata: annotations: {{- if .Values.storageClass }} volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} + {{- else if .Values.kube.storage_class.persistent }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} {{- else }} volume.alpha.kubernetes.io/storage-class: default {{- end }} diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index 7d67f89499..981a47d909 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -57,7 +57,7 @@ mariadb: enabled: true accessMode: ReadWriteOnce size: 1Gi -# storageClass: default + storageClass: uaa: protocol: https:// port: From f81a667160f31e03f68dcb3fb0a82dae69b8b382 Mon Sep 17 00:00:00 2001 From: Neil MacDougall Date: Tue, 31 Oct 2017 09:49:42 +0000 Subject: [PATCH 13/13] Fix indentation --- deploy/kubernetes/console/templates/pre-install.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/kubernetes/console/templates/pre-install.yaml b/deploy/kubernetes/console/templates/pre-install.yaml index 9ffb9206b4..610d01119a 100644 --- a/deploy/kubernetes/console/templates/pre-install.yaml +++ b/deploy/kubernetes/console/templates/pre-install.yaml @@ -9,7 +9,7 @@ metadata: {{- if .Values.storageClass }} volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} {{- else if .Values.kube.storage_class.persistent }} - volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} {{- else }} volume.alpha.kubernetes.io/storage-class: default {{- end }} @@ -37,7 +37,7 @@ metadata: {{- if .Values.storageClass }} volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} {{- else if .Values.kube.storage_class.persistent }} - volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} {{- else }} volume.alpha.kubernetes.io/storage-class: default {{- end }} @@ -59,7 +59,7 @@ metadata: {{- if .Values.storageClass }} volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} {{- else if .Values.kube.storage_class.persistent }} - volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} + volume.beta.kubernetes.io/storage-class: {{ .Values.kube.storage_class.persistent | quote }} {{- else }} volume.alpha.kubernetes.io/storage-class: default {{- end }}