Refactor to use OSRM (#11)

* Add new OSRM network construction stage

* Remove Valhalla variables

* Refactor times calc for OSRM

* Remove chunking

* Simplify snapping to nearest

* Fix ID order in output

* Remove Docker and Valhalla extras

* Drop old Vahalla tiles DVC stages

* Update OSRM network build

* Update OSRM network build

* Update paths to network files

* Update network fetch action

* Update more paths

* Chown docker outputs

* Move bash logic to dedicated action

* Tar as sudo

* Tar as sudo

* Update time calc Action

* Force install of old aws-cli version

* Use networks directly from R2

* Drop dependency on setup job

* Start Docker backend for jobs

* Parameterize state jobs

* Handle single coord edge case

* Move swap action to dedicated composite action

* Parameterize workflow by mode

* Enable zstd on network tarballs

* Remove files after tar

* Move bash scripts from dvc to dedicated scripts

* Consolidate times calc into one script

* Use swap action to clear disk

* Add executable bit to script

* Drop hash ingfor network tar file

* Remove containers on job completion

* Add lifecycle rules to buckets

* Add caching layer for OSRM builds

* Increase swap size, don't save local outputs

* Add debug action

* Revert "Add debug action"

This reverts commit 29d1ae7.

* Add pause to clear memory after network build

* Add sleep after Docker run

* Re-add chunking code to handle larger-than-memory outputs

* Update output file params

* Drop mode from chunk splits

* Add log upload on failure

* Poll docker logs for successful start

* Disable deep copy during concat

* Use concurrent futures for loop

* Install geos

* Refactor script to make public files

* Set single thread for duckdb

* Drop big states on map

* Drop group set unit tests

.github/actions/parse-gh-input/action.yaml

@@ -0,0 +1,45 @@
+---
+
+name: Parse GitHub Actions workflow input
+description: Convert strings to matrix-compatible JSON
+
+inputs:
+  param_path:
+    description: yq path to the verification parameter in params.yaml
+    required: true
+  param_override:
+    description: Comma-separated list overriding the parameter
+    required: true
+
+outputs:
+  param:
+    description: Parsed parameter value
+    value: ${{ steps.parse-input.outputs.param }}
+
+runs:
+  using: composite
+  steps:
+    - name: Parse input
+      id: parse-input
+      shell: bash
+      run: |
+        x=$(yq e -o=json '${{ inputs.param_path }}' ./data/params.yaml | jq -c -s .[])
+        echo "param=$x" >> $GITHUB_OUTPUT
+
+        # If override param is set, use that instead
+        x_parsed=($(echo "$x" | jq -r '.[]'))
+        if [ -n "${{ inputs.param_override }}" ]; then
+          override_parsed=($(echo "${{ inputs.param_override }}" | tr -d ' ' | tr ',' ' '))
+          for y in "${override_parsed[@]}"; do
+            if [[ ! " ${x_parsed[@]} " =~ " ${y} " ]]; then
+              echo "Error: Override param ${y} is not in the params for this workflow"
+              echo "Params include: ${x_parsed[@]}"
+              exit 1
+            fi
+          done
+          x_json=$(printf '%s\n' "${override_parsed[@]}" | jq -c -R . | jq -c -s .)
+          echo "Creating jobs for params: ${override_parsed[@]}"
+          echo "param=$x_json" >> $GITHUB_OUTPUT
+        else
+          echo "Creating jobs for params: ${x_parsed[@]}"
+        fi

.github/actions/prep-disk-and-swap/action.yaml

@@ -0,0 +1,48 @@
+---
+
+name: Increase swap and drive space
+description: Increase linux swap and drive space to prevent OoM
+inputs:
+  swap_override:
+    description: Swap file size in bytes, otherwise 90% of remaining space
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - name: Report disk space before modification
+      shell: bash
+      run: df -h
+
+    - name: Remove pre-installed runner software
+      shell: bash
+      run: |
+        # From: https://github.com/AdityaGarg8/remove-unwanted-software/blob/master/action.yml
+        sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup
+        sudo apt-get remove -y '^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' \
+          '^mongodb-.*' '^mysql-.*' '^azure-.*' '^google-.*' '^firefox.*' \
+          --fix-missing > /dev/null || true
+        sudo apt-get autoremove -y > /dev/null
+        sudo apt-get clean > /dev/null
+
+    - name: Report disk space after modification
+      shell: bash
+      run: df -h
+
+    - name: Increase swapfile
+      shell: bash
+      run: |
+        space_left=$(df /dev/root -B 1 --output=avail | grep -v Avail)
+        space_mult=0.9
+        space_alloc=$(echo "${space_left}*${space_mult}" | bc)
+        space_alloc_rnd=$(printf %.0f $(echo ${space_alloc}))
+        if [ -n "${{ inputs.swap_override }}" ]; then
+          space_alloc_rnd=${{ inputs.swap_override }}
+        fi
+        echo "Creating swapfile of ${space_alloc_rnd} bytes"
+        sudo swapoff -a
+        sudo fallocate -l ${space_alloc_rnd} /swapfile
+        sudo chmod 600 /swapfile
+        sudo mkswap /swapfile
+        sudo swapon /swapfile
+        sudo swapon --show

.github/actions/setup-cloudflare-s3/action.yaml

@@ -26,3 +26,13 @@ runs:
         echo "[cloudflare]" > ~/.aws/credentials
         echo "aws_access_key_id=${{ inputs.CLOUDFLARE_S3_API_ACCESS_KEY_ID }}" >> ~/.aws/credentials
         echo "aws_secret_access_key=${{ inputs.CLOUDFLARE_S3_API_SECRET_ACCESS_KEY }}" >> ~/.aws/credentials
+
+    # Cloudflare has some kind of dumb checksum issue with newer versions of the
+    # AWS CLI, so we need to install an older version
+    - name: Install AWS CLI
+      shell: bash
+      run: |
+        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.35.zip" -o "awscliv2.zip"
+        unzip -q awscliv2.zip
+        sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
+        aws --version