Skip to content

Commit

Permalink
feat(docker): Introduce reworked docker setup with jar (#1670)
Browse files Browse the repository at this point in the history
  • Loading branch information
koebi authored Mar 14, 2024
2 parents 3db9a0a + c5a92e9 commit d027dcd
Show file tree
Hide file tree
Showing 18 changed files with 916 additions and 289 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ ors-api/logs

ors-api/target/
ors-engine/target/
ors-docker/**

cgiar_provider/
cgiar_cache/
Expand Down
43 changes: 0 additions & 43 deletions .github/utils/config_conversion.sh

This file was deleted.

42 changes: 42 additions & 0 deletions .github/utils/yml_config_to_ors_config_conversion.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/bash

if [ "$#" -ne 2 ]; then
echo "Usage: $0 input.yaml output.yaml"
exit 1
fi

input_file=$1
output_file=$2

echo ""
echo "Copy $input_file to $output_file"
cp $input_file $output_file

###########################
### Replace parameters ####
###########################
echo ""
echo "Replace parameters:"

echo "- enable ors.engine.profiles.car"
yq -i '.ors.engine.profiles.car.enabled = true' "$output_file" || exit 1

echo "- set ors.engine.source_file to ors-api/src/test/files/heidelberg.osm.gz"
yq -i '.ors.engine.source_file = "ors-api/src/test/files/heidelberg.osm.gz"' "$output_file" || exit 1

###########################
### Convert input file ####
###########################
echo ""
echo "Converting input file:"
## Add # to the beginning of each line that is not empty or a comment
echo "- Comment everything"
sed -i '/^\s*[^#]/ s/^/#/' "$output_file" || exit 1

echo "- Uncomment ors, engine and source_file"
sed -i -e '/^#ors:/s/^#//' -e '/^#.*engine:/s/^#//' -e '/^#.*source_file:/s/^#//' "$output_file"

echo "- Uncomment subsequent lines for profiles.car.enabled in ors.engine"
sed -i -e '/^# profiles:/,/^# enabled:/ s/^#//' "$output_file"

echo "Parsing complete. Result saved to $output_file"
62 changes: 62 additions & 0 deletions .github/utils/yml_config_to_properties_conversion.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/bash

if [ "$#" -ne 2 ]; then
echo "Usage: $0 input.yaml output.properties"
exit 1
fi

input_file=$1
output_file=$2

echo ""
echo "Copy $input_file to $output_file"
cp $input_file $output_file

###########################
### Replace parameters ####
###########################
echo ""
echo "Replace parameters:"

echo "- enable ors.engine.profiles.car"
yq -i '.ors.engine.profiles.car.enabled = true' "$output_file" || exit 1

echo "- set ors.engine.source_file to ors-api/src/test/files/heidelberg.osm.gz"
yq -i '.ors.engine.source_file = "ors-api/src/test/files/heidelberg.osm.gz"' "$output_file" || exit 1


###########################
### Convert input file ####
###########################
echo ""
echo "Convert .yaml to .env/properties file:"
echo "- unwrap yaml structure to flat properties"
yq -i -o=props --unwrapScalar=false '.. | select(tag != "!!map" and tag != "!!seq") | ( (path | join(".")) + "=" + .)' "$output_file" || exit 1

## Add # to the beginning of each line that is not empty or a comment
echo "- Comment everything"
sed -i '/^\s*[^#]/ s/^/#/' "$output_file" || exit 1

echo "- Uncomment ors.engine.source_file and ors.engine.profiles.car.enabled"
sed -i -e '/^#ors.engine.source_file/s/^#//' -e '/^#ors.engine.profiles.car.enabled/s/^#//' "$output_file" || exit 1

############################
### Validate output file ###
############################
echo ""
echo "Validate output file:"
echo "- checking for ors.engine.source_file=ors-api/src/test/files/heidelberg.osm.gz"
return_value=$(sed -n '/^ors.engine.source_file=ors-api\/src\/test\/files\/heidelberg.osm.gz/p' $output_file)|| exit 1
if [ -z "$return_value" ]; then
echo "ors.engine.source_file=ors-api/src/test/files/heidelberg.osm.gz not found"
exit 1
fi
echo "- checking for ors.engine.profiles.car.enabled=true"
return_value=$(sed -n '/^ors.engine.profiles.car.enabled=true/p' $output_file) || exit 1
if [ -z "$return_value" ]; then
echo "ors.engine.profiles.car.enabled=true not found"
exit 1
fi

echo "Parsing complete. Result saved to $output_file"

21 changes: 21 additions & 0 deletions .github/utils/yml_config_validation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash

if [ "$#" -ne 1 ]; then
echo "Usage: $0 input.yaml"
exit 1
fi

input_file=$1

###########################
### Validate input file ###
###########################
echo "Validate input file:"
echo "- checking if the input file is a valid yaml file"
yq 'true' $input_file /dev/null || exit 1
# Fail if ors.engine.profiles.car.enabled='false' can't be found access with schema .result | select(.property_history != null) | .property_history | map(select(.event_name == "Sold"))[0].date'
echo "- checking if ors.engine.source_file exists and has 'source_file' property"
yq --exit-status '.ors.engine | has("source_file")' $input_file > /dev/null || exit 1
# For profiles section for car with enabled using yq and contains
echo "- checking if ors.engine.profiles.car exists and has 'enabled' property"
yq --exit-status '.ors.engine.profiles.car | has("enabled")' $input_file > /dev/null || exit 1
18 changes: 12 additions & 6 deletions .github/workflows/config-conversion-automation.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Automatic synchronization of application.yml to ors-config.yml
name: Automatic synchronization of application.yml to ors-config.yml and ors-config.env
on:
pull_request:
branches:
Expand All @@ -10,7 +10,7 @@ on:

jobs:
sync_config:
name: Synchronize changes in application.yml to ors-config.yml
name: Synchronize changes in application.yml to ors-config.yml and ors-config.env
runs-on: ubuntu-latest
permissions:
contents: write
Expand All @@ -20,10 +20,16 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- name: Convert application.yml to ors-config.yml
- name: yq - portable yaml processor
uses: mikefarah/[email protected]
- name: validate application.yml
run: .github/utils/yml_config_validation.sh ors-api/src/main/resources/application.yml
- name: Convert application.yml to ors-config.yml and ors-config.env
run: |
.github/utils/config_conversion.sh ors-api/src/main/resources/application.yml ors-config.yml
# Print yq version
yq --version
.github/utils/yml_config_to_ors_config_conversion.sh ors-api/src/main/resources/application.yml ors-config.yml
.github/utils/yml_config_to_properties_conversion.sh ors-api/src/main/resources/application.yml ors-config.env
- uses: MichaelsJP/git-auto-commit-action@v5
with:
commit_message: 'chore(config): automatic conversion of application.yml to ors-config.yml'

commit_message: 'chore(config): automatic conversion of application.yml to ors-config.yml and ors-config.env'
27 changes: 19 additions & 8 deletions .github/workflows/docker-build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
id: buildx
with:
install: true
Expand Down Expand Up @@ -95,7 +95,7 @@ jobs:
with:
platforms: ${{ matrix.platform }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
id: buildx
with:
install: true
Expand All @@ -116,16 +116,27 @@ jobs:
cache-from: type=gha
- name: Start container from previously build image and wait for successful checks
run: |
mkdir -p $(pwd)/graphs $(pwd)/conf
chown -R $UID $(pwd)/graphs $(pwd)/conf
docker run -it --user $UID -d -p 8080:8080 -v $(pwd)/graphs:/home/ors/ors-core/data/graphs -v $(pwd)/conf:/home/ors/ors-conf --name ors-instance ${{ needs.prepare_environment.outputs.test_image_name }}
mkdir -p $(pwd)/graphs $(pwd)/config
chown -R $UID $(pwd)/graphs $(pwd)/config
# Replace image: in the docker-compose.yml with the test image. The value of image: can vary.
sed -i "s|image:.*|image: ${{ needs.prepare_environment.outputs.test_image_name }}|" docker-compose.yml
# Start the first build with the docker-compose setup
docker compose up --build -d
# Wait for all logs to come in
sleep 5
docker compose logs ors-app
./.github/utils/url_check.sh 127.0.0.1 8080 /ors/v2/health 200 ${{ env.HEALTH_WAIT_TIME }}
# Stop the compose setup and continue with docker run
docker compose down
# Start the container with the test image and the raw docker run command
docker run -it --user $UID -d -p 8080:8082 -v $(pwd)/graphs:/home/ors/graphs -v $(pwd)/config:/home/ors/config --name ors-instance ${{ needs.prepare_environment.outputs.test_image_name }}
# Check for health to turn 200 after the graphs are build and spring-boot completely started
./.github/utils/url_check.sh 127.0.0.1 8080 /ors/v2/health 200 ${{ env.HEALTH_WAIT_TIME }}
# Check for correct preflight settings to avoid CORS issues with ORIGIN wildcard from the example config
./.github/utils/cors_check.sh 127.0.0.1 8080 /ors/v2/isochrones/geojson "https://example.org" 200 10
./.github/utils/cors_check.sh 127.0.0.1 8080 /ors/v2/isochrones/geojson "https://example.com" 200 10
echo "Adjust the config file and set ORIGIN to https://example.org"
sudo yq '.ors.cors.allowed_origins = "https://example.org, https://test.com, http://localhost:8080"' -i ./conf/ors-config.yml
sudo yq '.ors.cors.allowed_origins = "https://example.org, https://test.com, http://localhost:8080"' -i $(pwd)/config/ors-config.yml
# Restart the container to apply the config changes
docker restart ors-instance
# Request preflight with https://example.com and https://example.org to see if it gets applied correctly
Expand All @@ -135,8 +146,8 @@ jobs:
echo "Recreate the container to test if the graph can be properly read again"
docker stop ors-instance
docker container prune -f
chown -R $UID $(pwd)/graphs $(pwd)/conf
docker run -it --user $UID -d -p 8080:8080 -v $(pwd)/graphs:/home/ors/ors-core/data/graphs -v $(pwd)/conf:/home/ors/ors-conf --name ors-instance ${{ needs.prepare_environment.outputs.test_image_name }}
chown -R $UID $(pwd)/graphs $(pwd)/config
docker run -it --user $UID -d -p 8080:8082 -v $(pwd)/graphs:/home/ors/graphs -v $(pwd)/config:/home/ors/config -e ors.cors.allowed_origins=https://example.org --name ors-instance ${{ needs.prepare_environment.outputs.test_image_name }}
# Request preflight with https://example.com and https://example.org to see if it gets applied correctly
./.github/utils/cors_check.sh 127.0.0.1 8080 /ors/v2/isochrones/geojson "https://example.org" 200 50
# It should fail with http code 403 for https://example.com since the Origin is not covered.
Expand Down
63 changes: 50 additions & 13 deletions .github/workflows/publish-tagged-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,9 @@ jobs:
id: buildx
with:
install: true
- name: 'Get Previous tag'
id: previoustag
uses: "WyriHaximus/github-action-get-previous-tag@v1"
- name: 'Get next minor version'
id: semvers
uses: "WyriHaximus/github-action-next-semvers@v1"
with:
version: ${{ steps.previoustag.outputs.tag }}
- name: Get the release tag
id: get_tag
run: echo ::set-output name=tag::${{ github.event.release.tag_name }}
- name: Prepare
id: prepare
run: |
Expand All @@ -42,13 +37,11 @@ jobs:
DOCKER_IMAGE=openrouteservice/openrouteservice
CURRENT_VERSIONS=$(test_version)
LATEST_IMAGE_VERSION=${{ steps.previoustag.outputs.tag }}
HIGHEST_IMAGE_VERSION=${{ steps.semvers.outputs.v_mayor }}
LATEST_IMAGE_VERSION=${{ steps.get_tag.outputs.tag }}
DOCKER_PLATFORMS=linux/amd64,linux/arm64
BUILD_VERSION=true
TAGS_LATEST_VERSION="${DOCKER_IMAGE}:${LATEST_IMAGE_VERSION}"
TAGS_HIGHEST_VERSION="${DOCKER_IMAGE}:${HIGHEST_IMAGE_VERSION}"
echo "HIGHEST MAYOR VERSION: $TAGS_HIGHEST_VERSION"
TAGS_LATEST="${DOCKER_IMAGE}:latest"
Expand All @@ -60,7 +53,7 @@ jobs:
echo ::set-output name=build_version::${BUILD_VERSION}
echo ::set-output name=build_platforms::${DOCKER_PLATFORMS}
echo ::set-output name=buildx_tags_version::${TAGS_LATEST_VERSION}
echo ::set-output name=buildx_tags_version::${TAGS_LATEST_VERSION}
echo ::set-output name=buildx_tags_latest::${TAGS_LATEST}
- name: Login to DockerHub
if: ${{ steps.prepare.outputs.build_version == 'true' }}
Expand All @@ -72,7 +65,7 @@ jobs:
if: ${{ steps.prepare.outputs.build_version == 'true' }}
uses: actions/checkout@v4
with:
ref: ${{ steps.previoustag.outputs.tag }}
ref: ${{ steps.get_tag.outputs.tag }}
- name: Build and publish version if needed
if: ${{ steps.prepare.outputs.build_version == 'true' }}
uses: docker/build-push-action@v4
Expand All @@ -83,3 +76,47 @@ jobs:
platforms: ${{ steps.prepare.outputs.build_platforms }}
cache-from: type=gha
cache-to: type=gha,mode=max
build_and_publish_release_artifacts:
runs-on: ubuntu-latest
needs: build_and_publish_docker
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'maven'
cache-dependency-path: ./pom.xml
- name: Build JAR
run: |
mvn -q clean package -DskipTests
# Copy the JAR to the root directory
cp ors-api/target/ors.jar ors.jar
- name: Build WAR
run: |
mvn -q clean package -DskipTests -PbuildWar
# Copy the WAR to the root directory
cp ors-api/target/ors.war ors.war
- name: Get the release tag
id: get_tag
run: echo ::set-output name=tag::${{ github.event.release.tag_name }}
- name: Rewrite the docker-compose.yml image tag to the release tag
if: startsWith(github.ref, 'refs/tags/')
run: |
# Replace the image part
sed -i "s/local\/openrouteservice:latest/openrouteservice\/openrouteservice:${{ steps.get_tag.outputs.tag }}/g" docker-compose.yml
- name: Attach the files to the release
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
token: ${{ secrets.GITHUB_TOKEN }}
files: |
./ors.jar
./ors.war
./ors-config.yml
./ors-config.env
./docker-compose.yml
Loading

0 comments on commit d027dcd

Please sign in to comment.