[SERVER-2273] Migrate 3.4 to a new 4.0 instance

.github/CODEOWNERS

@@ -0,0 +1 @@
+@circleci-public/scale

README.md

@@ -0,0 +1,19 @@
+# CircleCI Scripts & Tools
+
+This repository contains various tools and scripts which supports CircleCI server installation and migrations.
+
+### [kots-exporter](./kots-exporter)
+
+`kots-exporter` script downloads the KOTS config from CircleCI `server 3.x` and create a helm value file which can be used in CircleCI `server 4.0` installation.
+
+### [migrate](./migrate)
+
+`migrate` directory holds collection of scripts which is resposible for backing-up the data from CircleCI `server 2.x` which can be restored into CircleCI `server 3.x` or `server 4.x` instance.
+
+### [passwords](./passwords)
+
+`passwords` directory contains [generate_password.sh](./passwords/generate_password.sh) script which generates various password or secrets for CircleCI `server 4.x` helm value file.
+
+### [support](./support/)
+
+`support` directory contains support-bundle template and instruction to generate support-bundle.

migrate-3-to-4/README.md

@@ -0,0 +1,20 @@
+# Migrating 3 to 4
+**Stop!** You probably want [kots-exporter](../kots-exporter/).  This script should only be used at the recomendation of your CircleCI Contact.
+
+## Prequsites
+1. There exists a 4.0 instance where reality check has successfully run.
+
+2. In the 3.4 environment, add the same [docker-registry secret](https://circleci.com/docs/server/installation/phase-2-core-services/#pull-images) that is used in the 4.0 environment
+
+## Using
+`./exporter.sh` will export all the datastores from a 3.4 instance onto your local machine.  `./restore.sh` will import the data now on your local machine into the 4.0 environment
+
+### Export data from 3.4
+1.  Set the kubectl context to the CircleCI 3.4 instance you are migrating **From**
+2.  Run: `./exporter.sh -n <namespace>`
+
+### Import data into 4.0
+1.  Set the Kubectl context to the CirlceCI 4.0 instance you are migrating **To**
+2.  Run: `./restore.sh <namespace>`
+3.  Update CircleCI 4.0 to point at the storage bucket from 3.4
+4.  Update Signing and Encryption keys they will be in `circleci_export/encryptkey` and `circleci_export/signkey`

migrate-3-to-4/bottoken.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+function reinject_bottoken() {
+    JOB_NAME=$(kubectl -n "$NAMESPACE" get jobs -o json | jq '.items[0].metadata.name' -r)
+    echo "Found job $JOB_NAME"
+    kubectl get job "${JOB_NAME}" -o json | jq 'del(.spec.template.metadata.labels."controller-uid") | del(.spec.selector)' > bottokenjob.json
+    kubectl replace --force -f bottokenjob.json
+}

migrate-3-to-4/exporter.sh

@@ -0,0 +1,211 @@
+#!/bin/bash
+set -e
+
+logFile="kots-exporter-script-$(date +%Y-%h-%d-%H%M%S).log"
+
+export BACKUP_DIR="circleci_export"
+
+help_init_options() {
+    echo ""
+    # Help message for Init menu
+    echo "Usage:"
+    echo "    ./exporter.sh [arguments]"
+    echo ""
+    echo "Arguments:"
+    echo "  -n|--namespace          (Required) k8s namespace where kots admin is installed"
+    echo "                           Defaults to 'circleci-server'"
+    echo "  -h|--help                Print help text"
+
+    echo ""
+    echo "Example :-"
+    echo "# Run kots-exporter with namespace"
+    echo "./exporter.sh -n <k8s-namespace>"
+    echo ""
+}
+
+check_prereq(){
+    echo ""
+    # check if kubectl is installed
+    if ! command -v kubectl version &> /dev/null
+    then
+        error_exit "kubectl could not be found."
+    fi
+
+    # check helm is installed
+    if ! command -v helm version &> /dev/null
+    then
+        error_exit "helm could not be found."
+    fi
+
+    # check yq is installed
+    if ! command -v yq -V &> /dev/null
+    then
+        error_exit "yq could not be found."
+    fi
+
+    # check jq is installed
+    if ! command -v jq -V &> /dev/null
+    then
+        error_exit "jq could not be found."
+    fi
+}
+
+create_folders(){
+    echo ""
+    echo "############ CREATING FOLDERS ################"
+
+    # Creating
+    rm -rf  "${path:?}/${BACKUP_DIR}" 2> /dev/null
+    mkdir -p "$path/${BACKUP_DIR}" && echo "output folder has been created."
+}
+
+execute_flyway_migration(){
+    echo ""
+    echo "############ RUNNING FLYWAY DB MIGRATION JOB ################"
+
+    echo "Checking if job/circle-migrator already ran -"
+    if kubectl get job/circle-migrator -n "$namespace" -o name > /dev/null 2>&1
+    then
+        echo "Job circle-migrator has already been run"
+        error_exit
+    fi
+
+    FRONTEND_POD=$(kubectl -n "$namespace" get pod -l app=frontend -o name | tail -1)
+    export FRONTEND_POD
+
+    echo "Fetching values from $FRONTEND_POD pod"
+    # shellcheck disable=SC2046
+    export $(kubectl -n "$namespace" exec "$FRONTEND_POD" -c frontend -- printenv | grep -Ew 'POSTGRES_USERNAME|POSTGRES_PORT|POSTGRES_PASSWORD|POSTGRES_HOST' | xargs )
+
+    echo "Creating job/circle-migrator -"
+    ( envsubst < "$path"/templates/circle-migrator.yaml | kubectl -n "$namespace" apply -f - ) \
+    || error_exit "Job circle-migrator creation error"
+
+    echo "Waiting job/circle-migrator to complete -"
+    if (kubectl wait job/circle-migrator --namespace "$namespace" --for condition="complete" --timeout=600s); then
+        echo "++++ DB Migration job is successful."
+        echo "Fetching pod logs -"
+        kubectl  -n "$namespace" logs "$(kubectl -n "$namespace" get pods -l app=circle-migrator -o name)" > "$path"/logs/circle-migrator.log
+        echo "Pod log is available at $path/logs/circle-migrator.log"
+        echo "Removing job/circle-migrator -"
+        kubectl delete job/circle-migrator --namespace "$namespace"
+    else
+        echo "Status wait timeout for job/circle-migrator, Check the Log via - kubectl logs pods -l app=circle-migrator"
+        echo "Job circle-migrator will delete automatically after 24 hours once complete."
+    fi
+}
+
+export_postgres() {
+    echo "Exporting PostgreSQL data"
+    PG_POD=$(kubectl -n "$namespace" get pods | grep postgresql | tail -1 | awk '{print $1}')
+    PG_PASSWORD=$(kubectl -n "$namespace" get secrets postgresql -o jsonpath="{.data.postgresql-password}" | base64 --decode)
+    kubectl -n "$namespace" exec -it "$PG_POD" -- bash -c "export PGPASSWORD='$PG_PASSWORD' && pg_dumpall -U postgres -c" > ${BACKUP_DIR}/circle.sql
+}
+
+##
+# Basic sanity check / smoke test to ensure that the postgresql export performed by
+# the export_postgres function contains what we expect it to contain
+##
+check_postgres() {
+    echo "Verifying postgres export file"
+    if [ -z "$(grep build_jobs ${BACKUP_DIR}/circle.sql | head -n1)" ]
+    then
+        echo "[FATAL] Something is wrong with the postgresql export file for 'conductor_production' database, please contact CircleCI support at [email protected] for further assistance."
+        exit 1
+    fi
+    if [ -z "$(grep contexts ${BACKUP_DIR}/circle.sql | head -n1)" ]
+    then
+        echo "[FATAL] Something is wrong with the postgresql export file for 'contexts_service_production' database, please contact CircleCI support at [email protected] for further assistance."
+        exit 1
+    fi
+    if [ -z "$(grep qrtz_blob_triggers ${BACKUP_DIR}/circle.sql | head -n1)" ]
+    then
+        echo "[FATAL] Something is wrong with the postgresql export file for 'cron_service_production' database, please contact CircleCI support at [email protected] for further assistance."
+        exit 1
+    fi
+    if [ -z "$(grep tasks ${BACKUP_DIR}/circle.sql | head -n1)" ]
+    then
+        echo "[WARN] 'vms' database was not correctly exported."
+    fi
+}
+
+export_mongo() {
+    echo "Exporting MongoDB data"
+    MONGO_POD="mongodb-0"
+    MONGODB_USERNAME="root"
+    MONGODB_PASSWORD=$(kubectl -n "$namespace" get secrets mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 --decode)
+    TEMP_DIR="/bitnami/circle-mongo"
+    kubectl -n "$namespace" exec -it "$MONGO_POD" -- bash -c "mkdir $TEMP_DIR"
+    kubectl -n "$namespace" exec -it "$MONGO_POD" -- bash -c "mongodump -u '$MONGODB_USERNAME' -p '$MONGODB_PASSWORD' --authenticationDatabase admin --db=circle_ghe --out=$TEMP_DIR"
+    kubectl -n "$namespace" cp $MONGO_POD:$TEMP_DIR ${BACKUP_DIR}/circle-mongo
+}
+
+export_vault() {
+    echo "Exporting Vault data"
+    VAULT_POD="vault-0"
+    kubectl -n "$namespace" cp -c vault "$VAULT_POD":file ${BACKUP_DIR}/file
+    tar cfz ${BACKUP_DIR}/vault-backup.tar.gz ${BACKUP_DIR}/file/
+    rm -rf ${BACKUP_DIR}/file
+}
+
+export_keys() {
+    echo "Exporting Keyset Keys"
+
+    SIGN_KEY=$(kubectl get deployment frontend -o json | jq '.spec.template.spec.containers[0].env | map(select(.name == "CIRCLE_SECRETS__KEYSET__SIGN"))[0].value')
+    echo "$SIGN_KEY" >> ${BACKUP_DIR}/signkey
+    ENCRYPTION_KEY=$(kubectl get deployment frontend -o json | jq '.spec.template.spec.containers[0].env | map(select(.name == "CIRCLE_SECRETS__KEYSET__CRYPT"))[0].value')
+    echo "$ENCRYPTION_KEY" >> ${BACKUP_DIR}/encryptkey
+
+
+}
+
+output_message(){
+    echo ""
+    echo "NOTE: After server 3.x to 4.x migration, You must rerun the Nomad terraform with modified value of 'server_endpoint' variable"
+}
+
+error_exit(){
+  msg="$*"
+
+  if [ -n "$msg" ] || [ "$msg" != "" ]; then
+    echo "------->> Error: $msg"
+  fi
+
+  kill $$
+}
+
+log_setup() {
+    path="$(cd "$(dirname "$0")" && pwd)"
+    mkdir -p "$path/logs"
+
+    exec > >(tee -a "$path/logs/$logFile") 2>&1
+
+    echo "Script Path: $path"
+}
+
+############ MAIN ############
+
+log_setup
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -n|--namespace)
+            namespace="$2";
+            shift ;;
+        -h|--help)
+            help_init_options;
+            exit 0 ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+check_prereq
+create_folders
+execute_flyway_migration
+export_postgres
+check_postgres
+export_mongo
+export_vault
+export_keys
+output_message

migrate-3-to-4/key.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+function key_reminder() {
+    echo ""
+    echo "###########################################"
+    echo "##   Encryption & Signing keys           ##"
+
+    echo "##   must be updated in your values.yaml.  ##"
+    echo "##   Then perform a helm upgrade.          ##"
+
+    echo "###########################################"
+    echo ""
+    echo "You may find your keys here:  $(pwd)/circleci_export"
+}

migrate-3-to-4/mongo.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+function import_mongo() {
+    echo "Importing Mongo";
+
+    MONGO_POD="mongodb-0"
+    MONGODB_USERNAME="root"
+    MONGODB_PASSWORD=$(kubectl -n "$NAMESPACE" get secrets mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 --decode)
+
+    kubectl -n "$NAMESPACE" exec "$MONGO_POD" -- mkdir -p /tmp/backups/
+    kubectl -n "$NAMESPACE" cp -v=2 "$MONGO_BU" "$MONGO_POD":/tmp/backups/
+
+    kubectl -n "$NAMESPACE" exec "$MONGO_POD" -- mongorestore --drop -u "$MONGODB_USERNAME" -p "$MONGODB_PASSWORD" --authenticationDatabase admin /tmp/backups/circle-mongo;
+    kubectl -n "$NAMESPACE" exec "$MONGO_POD" -- rm -rf /tmp/backups
+}

migrate-3-to-4/postgres.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+function import_postgres() {
+    echo 'Importing Postgres'
+
+    PG_POD=$(kubectl -n "$NAMESPACE" get pods | grep postgresql | tail -1 | awk '{print $1}')
+
+    PG_PASSWORD=$(kubectl -n "$NAMESPACE" get secrets postgresql -o jsonpath="{.data.postgres-password}" | base64 --decode)
+
+    # Server 3 and 4 both have a user named `postgres`.
+    # The postgres dump will drop all resources before trying to create new ones, including the postgres user.
+    # Remove the lines that would delete the postgres user.
+    # this is not a problem when migrating from 2.19 becuase 2.19's username was 'circle'
+    sed -i ".bak" '/DROP ROLE postgres/d' "$PG_BU"/circle.sql
+    sed -i ".bak" '/CREATE ROLE postgres/d' "$PG_BU"/circle.sql
+    sed -i ".bak" '/ALTER ROLE postgres WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICATION BYPASSRLS PASSWORD/d' "$PG_BU"/circle.sql
+
+    # Note: This import assumes `pg_dumpall -c` was run to drop tables before ...importing into them.
+    kubectl -n "$NAMESPACE" exec -i "$PG_POD" -- env PGPASSWORD="$PG_PASSWORD" psql -U postgres < "$PG_BU"/circle.sql
+}

migrate-3-to-4/preflight.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+##
+# Preflight checks
+#   make sure the namespace exists
+##
+function preflight_checks() {
+    echo "Starting preflight checks"
+    if [ -z "$NAMESPACE" ]
+    then
+        echo "Syntax: restore.sh <namespace>"
+        exit 1
+    elif [ ! "$(which kubectl)" ]
+    then
+        echo ... "'kubectl' not found"
+        exit 1
+    elif [ ! "$(which jq)" ]
+    then
+        echo ... "'jq' not found"
+        exit 1
+    elif [ "$(kubectl get namespace --no-headers "$NAMESPACE" | wc -w)" -eq 0 ]
+    then
+        echo "Namespace '$NAMESPACE' not found."
+        exit 1
+    elif [ ! -s "$PG_BU"/circle.sql ]
+    then
+        echo "Postgres data at '$PG_BU/circle.sql' not found (or is empty)"
+        exit 1
+    elif [ ! -s "$MONGO_BU" ]
+    then
+        echo "Mongo data at '$MONGO_BU' not found (or is empty)"
+        exit 1
+    elif [[ ! -s "$VAULT_BU" && $(du -s "$VAULT_BU" 2>/dev/null | awk '{print $1}') -lt 5 ]]
+    then
+        echo "Vault data at '$VAULT_BU' not found (or is empty)"
+        exit 1
+    fi
+    echo "Finishing preflight checks"
+}