update-example-datasets-mt #65
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: update-example-datasets-mt | |
| on: | |
| schedule: | |
| - cron: "45 1 * * *" | |
| workflow_dispatch: {} | |
| concurrency: update-example-datasets-mt | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-west-2 | |
| jobs: | |
| update: | |
| name: Update example datasets | |
| runs-on: ubuntu-latest | |
| steps: | |
| ########################################### | |
| # Install dependencies | |
| ########################################### | |
| - name: Install kamu | |
| shell: bash | |
| run: | | |
| curl -s "https://get.kamu.dev" | sh | |
| - name: Print info | |
| shell: bash | |
| run: | | |
| echo "Work dir: $PWD" | |
| echo "Podman version:" | |
| podman version | |
| echo "AWS CLI version:" | |
| aws --version | |
| echo "Kamu version:" | |
| kamu --version | |
| ########################################### | |
| # Caching | |
| ########################################### | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| workspace | |
| # TODO: Currently cache action will not update the cache upon hit and there is no option to do so | |
| # Below uses a workaround suggested here: https://github.com/actions/cache/issues/342#issuecomment-673371329 | |
| key: example-datasets-mt-${{ github.run_id }} | |
| restore-keys: | | |
| example-datasets-mt- | |
| ########################################### | |
| # Checkout repo | |
| ########################################### | |
| - name: Checkout kamu-contrib | |
| uses: actions/checkout@v3 | |
| with: | |
| path: kamu-contrib | |
| ########################################### | |
| # Update | |
| ########################################### | |
| - name: Download fresh datasets from S3 | |
| shell: bash | |
| run: | | |
| mkdir -p workspace | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets-mt/init-from-s3.py | |
| kamu list --all-accounts -w | |
| - name: Push to S3 | |
| shell: bash | |
| run: | | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets-mt/push-to-s3.py | |
| # Keep demo env up-to-date with flowing data | |
| - name: Push to Demo Env | |
| shell: bash | |
| env: | |
| S3_TARGET_URL: "s3://datasets.demo.kamu.dev/" | |
| run: | | |
| cd workspace | |
| ../kamu-contrib/.ci/example-datasets-mt/push-to-s3.py |