update-example-datasets.yaml

name: update-example-datasets
on:
  schedule:
    - cron: "45 0 * * *"
  workflow_dispatch: {}
concurrency: update-example-datasets
env:
  # AWS credentials
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-west-2
  # Dataset pull credentials
  ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }}
jobs:
  update:
    name: Update example datasets
    runs-on: ubuntu-latest
    steps:

      ###########################################
      # Install dependencies
      ###########################################

      - name: Install kamu
        shell: bash
        run: |
          curl -s "https://get.kamu.dev" | sh

      - name: Print info
        shell: bash
        run: |
          echo "Work dir: $PWD"

          echo "Podman version:"
          podman version

          echo "AWS CLI version:"
          aws --version

          echo "Kamu version:"
          kamu --version

      ###########################################
      # Caching
      ###########################################

      - uses: actions/cache@v3
        with:
          path: |
            workspace
          # TODO: Currently cache action will not update the cache upon hit and there is no option to do so
          # Below uses a workaround suggested here: https://github.com/actions/cache/issues/342#issuecomment-673371329
          key: example-datasets-st-${{ github.run_id }}
          restore-keys: |
            example-datasets-st-

      ###########################################
      # Checkout repo
      ###########################################

      - name: Checkout kamu-contrib
        uses: actions/checkout@v3
        with:
          path: kamu-contrib

      ###########################################
      # Update
      ###########################################

      - name: Init workspace from S3
        shell: bash
        run: |
          mkdir -p workspace
          cd workspace
          ../kamu-contrib/.ci/example-datasets/init-from-s3.py

          echo "State before update:"
          kamu list -w

      - name: Update datasets
        shell: bash
        run: |
          cd workspace
          ../kamu-contrib/.ci/example-datasets/update.py

          echo "State after update:"
          kamu list -w

      - name: Push to S3
        shell: bash
        run: |
          cd workspace
          ../kamu-contrib/.ci/example-datasets/push-to-s3.py