diff --git a/.github/workflows/build_and_push_docker.yaml b/.github/workflows/build_and_push_docker.yaml index d6de80b95c..c2bd10ae7a 100644 --- a/.github/workflows/build_and_push_docker.yaml +++ b/.github/workflows/build_and_push_docker.yaml @@ -2,6 +2,10 @@ name: Build and Push Docker Image on: push: + branches: [ main ] + paths: + - '.github/image/Dockerfile' + pull_request: paths: - '.github/image/Dockerfile' workflow_dispatch: @@ -27,9 +31,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # Build and push pr specific images when there's changes to Dockerfile as part of a PR + # Update the 'latest' tag when the change hits master + - name: Set Docker tag + id: docker_tag + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "tag=${REGISTRY}/${IMAGE_NAME}:pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + else + echo "tag=${REGISTRY}/${IMAGE_NAME}:latest" >> $GITHUB_OUTPUT + fi + - name: Build and push Docker image uses: docker/build-push-action@v6 with: context: .github/image push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ No newline at end of file + tags: ${{ steps.docker_tag.outputs.tag }} \ No newline at end of file diff --git a/.github/workflows/test_scala_and_python.yaml b/.github/workflows/test_scala_and_python.yaml index 62bd22b3a4..4776566cdb 100644 --- a/.github/workflows/test_scala_and_python.yaml +++ b/.github/workflows/test_scala_and_python.yaml @@ -6,11 +6,29 @@ on: pull_request: branches: [ main ] +# Note on the container image we're using in the various jobs below: +# If this is in a PR we use the pr specific docker image (as there might be docker file changes) +# Else we use the 'latest' image jobs: + determine-image: + runs-on: ubuntu-latest + outputs: + container_image: ${{ steps.set-image.outputs.container_image }} + steps: + - id: set-image + run: | + REPOSITORY="${{ github.repository }}" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "container_image=ghcr.io/${REPOSITORY,,}-ci:pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + else + echo "container_image=ghcr.io/${REPOSITORY,,}-ci:latest" >> $GITHUB_OUTPUT + fi + python_tests: + needs: determine-image runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -50,8 +68,9 @@ jobs: other_spark_tests: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -69,8 +88,9 @@ jobs: join_spark_tests: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -88,8 +108,9 @@ jobs: mutation_spark_tests: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -107,8 +128,9 @@ jobs: fetcher_spark_tests: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -124,10 +146,11 @@ jobs: export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" sbt "spark/testOnly ai.chronon.spark.test.FetcherTest" - scala_compile_fmt_fix : + scala_compile_fmt_fix: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -150,8 +173,9 @@ jobs: other_scala_tests: runs-on: ubuntu-latest + needs: determine-image container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ${{ needs.determine-image.outputs.container_image }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }}