From c8b789f854e8575c54c7744593d4fa26435b3173 Mon Sep 17 00:00:00 2001 From: Pieter van der Meulen Date: Thu, 21 Sep 2023 14:51:36 +0200 Subject: [PATCH 1/3] Add script to do all the (first time) database initialisation The script vcan be safely run multiple times, e.g. to upgrade the database schema for new versions, or to update the middleware configuration --- stepup/init-db.sh | 23 +++++++++++++++++ stepup/middleware/middleware-push-config.sh | 25 ++++++++++--------- .../middleware/middleware-push-institution.sh | 21 ++++++++-------- .../middleware/middleware-push-whitelist.sh | 21 ++++++++-------- 4 files changed, 58 insertions(+), 32 deletions(-) create mode 100755 stepup/init-db.sh mode change 100644 => 100755 stepup/middleware/middleware-push-config.sh mode change 100644 => 100755 stepup/middleware/middleware-push-institution.sh mode change 100644 => 100755 stepup/middleware/middleware-push-whitelist.sh diff --git a/stepup/init-db.sh b/stepup/init-db.sh new file mode 100755 index 0000000..85d531b --- /dev/null +++ b/stepup/init-db.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Get this script's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +echo "Initializing middleware and gateway databases" +docker compose exec middleware /var/www/html/bin/console doctrine:migrations:migrate --env=prod --em=deploy --no-interaction + +echo "Initializing webauthn database" +docker compose exec webauthn /var/www/html/bin/console doctrine:migrations:migrate --env=prod --no-interaction + +echo "Pushing middleware configuration" +sh "${DIR}/middleware/middleware-push-config.sh" + +echo "Pushing middleware whitelist" +sh "${DIR}/middleware/middleware-push-whitelist.sh" + +echo "Pushing middleware institution configuration" +sh "${DIR}/middleware/middleware-push-institution.sh" + +echo "Done" \ No newline at end of file diff --git a/stepup/middleware/middleware-push-config.sh b/stepup/middleware/middleware-push-config.sh old mode 100644 new mode 100755 index d452826..056ff9e --- a/stepup/middleware/middleware-push-config.sh +++ b/stepup/middleware/middleware-push-config.sh @@ -1,13 +1,13 @@ #!/bin/bash -CWD=$(pwd) +# Get this script's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" function error_exit { echo "${1}" - if [ -n "${TMP_FILE}" -a -d "${TMP_FILE}" ]; then + if [ -n "${TMP_FILE}" ] && [ -d "${TMP_FILE}" ]; then rm "${TMP_FILE}" fi - cd ${CWD} exit 1 } @@ -18,20 +18,21 @@ if [ $? -ne "0" ]; then error_exit "Could not create temp file" fi -echo "Pushing new config to: http://middleware.dev.openconext.local/management/configuration" +echo "Pushing new middleware configuration to: http://middleware.dev.openconext.local/management/configuration" +echo "Reading middleware configuration from: ${DIR}/middleware-config.json"; -http_response=$(curl -k --write-out %{http_code} --output ${TMP_FILE} -XPOST -s \ +http_response=$(curl -k --write-out %\{http_code\} --output "${TMP_FILE}" -XPOST -s \ -u management:secret \ -H "Accept: application/json" \ -H "Content-type: application/json" \ - -d @middleware-config.json \ + -d "@${DIR}/middleware-config.json" \ https://middleware.dev.openconext.local/management/configuration) +res=$? -output=$(cat ${TMP_FILE}) -rm ${TMP_FILE} -echo $output +output=$(cat "${TMP_FILE}") +rm "${TMP_FILE}" +echo "$output" -res=$? if [ $res -ne "0" ]; then error_exit "Curl failed with code $res" fi @@ -43,8 +44,8 @@ fi # On success JSON output should start with: {"status":"OK" ok_count=$(echo "${output}" | grep -c "status") -if [ $ok_count -ne "1" ]; then +if [ "$ok_count" -ne "1" ]; then error_exit "Expected one JSON \"status: OK\" in response, found $ok_count" fi -echo "OK. New config pushed"% +echo "OK. New config pushed" diff --git a/stepup/middleware/middleware-push-institution.sh b/stepup/middleware/middleware-push-institution.sh old mode 100644 new mode 100755 index 282c1c4..bee6c25 --- a/stepup/middleware/middleware-push-institution.sh +++ b/stepup/middleware/middleware-push-institution.sh @@ -1,13 +1,13 @@ #!/bin/bash -CWD=$(pwd) +# Get this script's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" function error_exit { echo "${1}" - if [ -n "${TMP_FILE}" -a -d "${TMP_FILE}" ]; then + if [ -n "${TMP_FILE}" ] && [ -d "${TMP_FILE}" ]; then rm "${TMP_FILE}" fi - cd ${CWD} exit 1 } @@ -19,19 +19,20 @@ if [ $? -ne "0" ]; then fi echo "Pushing new institution configuration to: https://middleware.dev.openconext.local/management/institution-configuration" +echo "Reading institution configuration from: ${DIR}/middleware-institution.json"; -http_response=$(curl -k --write-out %{http_code} --output ${TMP_FILE} -XPOST -s \ +http_response=$(curl -k --write-out %\{http_code\} --output "${TMP_FILE}" -XPOST -s \ -u management:secret \ -H "Accept: application/json" \ -H "Content-type: application/json" \ - -d @middleware-institution.json \ + -d "@${DIR}/middleware-institution.json" \ https://middleware.dev.openconext.local/management/institution-configuration) +res=$? -output=$(cat ${TMP_FILE}) -rm ${TMP_FILE} -echo $output +output=$(cat "${TMP_FILE}") +rm "${TMP_FILE}" +echo "$output" -res=$? if [ $res -ne "0" ]; then error_exit "Curl failed with code $res" fi @@ -43,7 +44,7 @@ fi # On success JSON output should start with: {"status":"OK" ok_count=$(echo "${output}" | grep -c "status") -if [ $ok_count -ne "1" ]; then +if [ "$ok_count" -ne "1" ]; then error_exit "Expected one JSON \"status: OK\" in response, found $ok_count" fi diff --git a/stepup/middleware/middleware-push-whitelist.sh b/stepup/middleware/middleware-push-whitelist.sh old mode 100644 new mode 100755 index a5402ce..7615dfb --- a/stepup/middleware/middleware-push-whitelist.sh +++ b/stepup/middleware/middleware-push-whitelist.sh @@ -1,13 +1,13 @@ #!/bin/bash -CWD=$(pwd) +# Get this script's directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" function error_exit { echo "${1}" - if [ -n "${TMP_FILE}" -a -d "${TMP_FILE}" ]; then + if [ -n "${TMP_FILE}" ] && [ -d "${TMP_FILE}" ]; then rm "${TMP_FILE}" fi - cd ${CWD} exit 1 } @@ -19,18 +19,19 @@ if [ $? -ne "0" ]; then fi echo "Pushing new institution whitelist to: http://middleware.dev.openconext.local/management/whitelist/replace" +echo "Reading whitelist from: ${DIR}/middleware-whitelist.json"; -http_response=$(curl -k --write-out %{http_code} --output ${TMP_FILE} -XPOST -s \ +http_response=$(curl -k --write-out %\{http_code\} --output "${TMP_FILE}" -XPOST -s \ -u management:secret -H "Accept: application/json" \ -H "Content-type: application/json" \ - -d @middleware-whitelist.json \ + -d "@${DIR}/middleware-whitelist.json" \ https://middleware.dev.openconext.local/management/whitelist/replace) +res=$? -output=$(cat ${TMP_FILE}) -rm ${TMP_FILE} -echo $output +output=$(cat "${TMP_FILE}") +rm "${TMP_FILE}" +echo "$output" -res=$? if [ $res -ne "0" ]; then error_exit "Curl failed with code $res" fi @@ -42,7 +43,7 @@ fi # On success JSON output should start with: {"status":"OK" ok_count=$(echo "${output}" | grep -c "status") -if [ $ok_count -ne "1" ]; then +if [ "$ok_count" -ne "1" ]; then error_exit "Expected one JSON \"status: OK\" in response, found $ok_count" fi From f15fc55023b4c963d17c60e80f686960ef4f0698 Mon Sep 17 00:00:00 2001 From: Pieter van der Meulen Date: Thu, 21 Sep 2023 17:22:09 +0200 Subject: [PATCH 2/3] Fix symfony cache ownership after running console command as root --- stepup/init-db.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stepup/init-db.sh b/stepup/init-db.sh index 85d531b..d53ffd4 100755 --- a/stepup/init-db.sh +++ b/stepup/init-db.sh @@ -7,9 +7,11 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "Initializing middleware and gateway databases" docker compose exec middleware /var/www/html/bin/console doctrine:migrations:migrate --env=prod --em=deploy --no-interaction +docker compose exec middleware chown -R www-data /var/www/html/var/cache echo "Initializing webauthn database" docker compose exec webauthn /var/www/html/bin/console doctrine:migrations:migrate --env=prod --no-interaction +docker compose exec webauthn chown -R www-data /var/www/html/var/cache echo "Pushing middleware configuration" sh "${DIR}/middleware/middleware-push-config.sh" From 91b1171d7a5691eb31c0060d19d15b030b29f9af Mon Sep 17 00:00:00 2001 From: Pieter van der Meulen Date: Fri, 22 Sep 2023 13:36:59 +0200 Subject: [PATCH 3/3] Simplify setup instructions --- stepup/README.md | 54 +++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/stepup/README.md b/stepup/README.md index cc995e4..9dca087 100644 --- a/stepup/README.md +++ b/stepup/README.md @@ -1,27 +1,27 @@ # OpenConext Stepup config for development and testing purposes -This folder contains configuration that is needed to get an OpenConext Stepup developmentenvironment up and running. +This folder contains configuration that is needed to get an OpenConext Stepup development environment up and running. The main configuration for each app can be found in each application repository. The extension .dist is used for that (parameters.yaml.dist for instance). The idea is that these configuration files contain everything necessary to get a development environment up and running and that there is no need to change that. The containers will have a working copy of those .dist configuration files. This repository contains the docker-compose.yml to get all containers that are used for development up and running. -* The application containers +* The application containers * A loadbalancer in front of it * A MariaDB container for the databases. -A SQL in the directory dbschema which creates databases and users needed for OpenConext Stepup development is mounted in the MariaDB container. +A SQL in the directory dbschema which creates databases and users needed for OpenConext Stepup development is mounted in the MariaDB container. -The application config directories contain the SAML key material. Those are not shipped with the application containers to prevent accidental usage of that key material in a production environment. The docker-compose mounts the application specific directory in /config. +The application config directories contain the SAML key material. Those are not shipped with the application containers to prevent accidental usage of that key material in a production environment. The docker-compose mounts the application specific directory in /config. -# Getting everything up and running +## Getting everything up and running -First, you need to create an entry in your hosts file (/etc/hosts on *nix systems) +First, you need to create an entry in your hosts file (`/etc/hosts` on *nix systems) -``` +```text 127.0.0.1 selfservice.dev.openconext.local webauthn.dev.openconext.local ssp.dev.openconext.local gateway.dev.openconext.local middleware.dev.openconext.local ra.dev.openconext.local demogssp.dev.openconext.local ``` -Secondly you need to create the `stepup/gateway/surfnet_yubikey.yaml` filewith your Yubikey API credentials. If you do not have API credentials, you can get them at . You require a Yubikey to get an API key. +Secondly you need to create the `stepup/gateway/surfnet_yubikey.yaml` filew ith your Yubikey API credentials. If you do not have API credentials, you can get them at . You require a Yubikey to get an API key. ```yaml surfnet_yubikey_api_client: @@ -30,36 +30,34 @@ surfnet_yubikey_api_client: client_secret: 'YOUR_SECRET' ``` -You should then get the apps initialised You can then bring up the containers using docker compose: -Initialise the middelware database: -``` -docker compose exec middleware /var/www/html/bin/console doctrine:migrations:migrate --env=prod --em=deploy +```text docker-compose up -d ``` -Then the webauthn db -``` -docker compose exec webauthn /var/www/html/bin/console doctrine:migrations:migrate --env=prod -``` +You should then get the apps initialised. -Then you will need to provision the middleware config: -``` -cd middleware -sh middleware-push-config.sh -sh middleware-push-whitelist.sh -sh middleware-push-institution.sh -``` -Then, bootstrap the SRAA. For this, you will need to have a Yubikey. Replace Yubikey_ID with the number that is printed on your yubikey. It should be 8 characters. If it is less, prepend it with 0's +Initialise the middelware, gateway and webauthn database schema's and push the configuration of the middleware to the database: + +```shell +./init-db.sh ``` -docker compose exec middleware /var/www/html/bin/console middleware:bootstrap:identity-with-yubikey urn:collab:person:dev.openconext.local:admin dev.openconext.local "Your Name" Your@email nl_NL Yubikey_ID + +Then, bootstrap the SRAA user. For this, you will need to have a Yubikey. Replace `Yubikey_ID` in the command below with the serial number of your your Yubikey. This number is printed on your Yubikey. You can also get it by converting the first 12 characters from an OTP from your Yubikey from ModHex to decimal using . The serial number must be at least 8 digits long. If it has less digits, prepend it with 0's. + +This associates the admin account with your Yubikey. The account has SRAA rights. You can then use your Yubikey to log in as user `admin` with password `admin`. Note that you must run this command before the first time you log in with the admin account to the SA or RA, otherwise this command will fail because the admin identity already exists. + +```shell +docker compose exec middleware /var/www/html/bin/console middleware:bootstrap:identity-with-yubikey urn:collab:person:dev.openconext.local:admin dev.openconext.local "Your Name" Your@email en_GB Yubikey_ID ``` -Mailcatcher is included. You can view the email by going to http://localhost:1080 +You can now login to the self-service portal at and login to the RA portal at using your admin/admin account. + +Mailcatcher is included. You can view the email by going to -A SimpleSAMLPHP sp is included. It can be accessed at https://ssp.dev.openconext.local/simplesaml/sp.php +A SimpleSAMLphp SP for testing authentication from an SP is included. It can be accessed at -# Starting a project in development mode +## Starting a project in development mode You can mount your local directory inside a development container which contains the correct node and composer versions for your project. To do so use the script start-dev-env.sh. It takes two parameters: the service name and the local directory to mount. Example: start-dev-env.sh webauthn /home/dan/Stepup-webauthn (the recommended way would be to use absolute paths). The startup script uses these two parameters to read the docker compose override file from the service's directory and replace the code path in that file (by reading it as an env var)