Skip to content

Commit

Permalink
code review changes: add deploy-type option
Browse files Browse the repository at this point in the history
  • Loading branch information
neel-astro committed Sep 13, 2024
1 parent 43e1c34 commit 203064b
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 64 deletions.
55 changes: 55 additions & 0 deletions .github/workflows/e2e/get_bundle_info/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
name: Get Deployment Bundle Info
description: Get deployment bundle info from Astronomer API
inputs:
deployment_id:
description: The deployment ID
required: true
organization_id:
description: The organization ID
required: true
astro_api_token:
description: The Astronomer API token
required: true
astronomer_host:
description: The Astronomer host
required: true
search_term:
description: The search term
required: true
expected_status_code:
description: The expected status code
default: 200

outputs:
desired_bundle_version:
description: The desired bundle version
value: ${{ steps.get-deployment-info.outputs.desired_bundle_version }}
bundle_type:
description: The bundle type
value: ${{ steps.get-deployment-info.outputs.bundle_type }}

runs:
using: "composite"
steps:
- name: Get Deployment Info
id: get-deployment-info
shell: bash
run: |
STATUS_CODE=$(curl -s -w "%{http_code}" -o response.json -H "Authorization: Bearer ${{ inputs.astro_api_token }}" "https://api.${{ inputs.astronomer_host }}/v1alpha1/organizations/${{ inputs.organization_id }}/deployments/${{ inputs.deployment_id }}/deploys?search=${{ inputs.search_term }}")
cat response.json
if [[ $STATUS_CODE -ne ${{ inputs.expected_status_code }} ]]; then
echo "Failed to get expected status code from GET Deployment API. Status code: $STATUS_CODE"
exit 1
fi
if [[ $(cat response.json | jq -r '.deploys | length') -eq 0 ]]; then
echo "No deploys found for search term: ${{ inputs.search_term }}"
exit 1
fi
if [[ $(cat response.json | jq -r '.deploys[0].type') != "BUNDLE" ]]; then
echo "No deploy of type bundle found: ${{ inputs.search_term }}"
exit 1
fi
desired_bundle_version=$(cat response.json | jq -r '.deploys[0].bundles[0].desiredVersion')
bundle_type=$(cat response.json | jq -r '.deploys[0].bundles[0].bundleType')
echo "desired_bundle_version=$desired_bundle_version" >> $GITHUB_OUTPUT
echo "bundle_type=$bundle_type" >> $GITHUB_OUTPUT
100 changes: 51 additions & 49 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ jobs:

- name: Mock git commands
run: |
mv e2e-setup/mocks/git.sh /usr/local/bin/git
mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Install dependencies
Expand Down Expand Up @@ -162,6 +162,8 @@ jobs:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
pytest: true
action: deploy
deploy-type: image-and-dags
root-folder: e2e-setup/astro-project

- name: Get Deployment Info After Test
Expand Down Expand Up @@ -224,6 +226,8 @@ jobs:
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy
deploy-type: image-and-dags
image-name: custom-image:latest
root-folder: e2e-setup/astro-project

Expand Down Expand Up @@ -262,7 +266,7 @@ jobs:

- name: Mock git commands
run: |
mv e2e-setup/mocks/git.sh /usr/local/bin/git
mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Install dependencies
Expand All @@ -275,15 +279,6 @@ jobs:
- name: Set CLI context
run: astro context switch ${{ secrets.ASTRONOMER_HOST }}

- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}

- name: DBT Deploy to Astro
uses: ./
with:
Expand All @@ -292,24 +287,32 @@ jobs:
parse: true
root-folder: e2e-setup/dbt
deploy-type: dbt
description: "test-dbt-deploy-action"

- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
- name: Get DBT bundle info
id: get-dbt-bundle-info
uses: ./.github/workflows/e2e/get_bundle_info
with:
deployment_id: ${{ matrix.deployment_id }}
deployment_id: ${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
search_term: "test-dbt-deploy-action" # We are passing description in the dbt deploy action and using that as search term over here because that's the only search term we have

- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
- name: Validate DBT Deploy Action
id: validate-dbt-deploy-action
shell: bash
run: |
if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then
echo "DBT Deploy Action validation failed: DBT bundle not found"
exit 1
fi
if [[ "${{ steps.get-dbt-bundle-info.outputs.bundle_type }}" != "dbt" ]]; then
echo "DBT Deploy Action validation failed: DBT bundle not found"
exit 1
fi
echo "DBT Deploy Action validation succeeded: DBT bundle found"
exit 0
# Deployment preview tests can run in parallel to above tests since it operates on a different deployment, so it won't interfere with the above tests
deployment-preview-test:
Expand Down Expand Up @@ -387,6 +390,7 @@ jobs:
deployment-id: ${{ steps.create-deployment-preview.outputs.preview-id }}
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
deploy-type: image-and-dags
root-folder: e2e-setup/astro-project
preview-name: test-preview-${{ matrix.deployment_id }}

Expand All @@ -410,8 +414,7 @@ jobs:

- name: Mock git commands for DAG Deploy
run: |
cp /usr/local/bin/git /usr/local/bin/git-original
mv e2e-setup/mocks/git.sh /usr/local/bin/git
mv e2e-setup/mocks/dag-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Get Deployment Info Before DAG Deploy Test
Expand All @@ -431,14 +434,10 @@ jobs:
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
parse: true
deploy-type: dags-only
root-folder: e2e-setup/astro-project
preview-name: test-preview-${{ matrix.deployment_id }}

- name: Revert mock git commands
run: |
mv /usr/local/bin/git-original /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: Get Deployment Info After DAG Deploy Test
id: get-deployment-after-dag-deploy
uses: ./.github/workflows/e2e/get_deployment_info
Expand All @@ -458,14 +457,10 @@ jobs:
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }}

- name: Get Deployment Info Before DBT Deploy Test
id: get-deployment-before-dbt-deploy
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
- name: Mock git commands for DAG Deploy
run: |
mv e2e-setup/mocks/dbt-deploy-git.sh /usr/local/bin/git
chmod +x /usr/local/bin/git
- name: DBT Deploy to Astro
id: deployment-preview-dbt-deploy
Expand All @@ -475,27 +470,34 @@ jobs:
workspace: ${{ secrets.WORKSPACE_ID }}
action: deploy-deployment-preview
deploy-type: dbt
root-folder: e2e-setup/astro-project
description: "test-dbt-deploy-action"
root-folder: e2e-setup/dbt
preview-name: test-preview-${{ matrix.deployment_id }}

- name: Get Deployment Info After DBT Deploy Test
id: get-deployment-after-dbt-deploy
uses: ./.github/workflows/e2e/get_deployment_info
- name: Get DBT bundle info
id: get-dbt-bundle-info
uses: ./.github/workflows/e2e/get_bundle_info
with:
deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
search_term: "test-dbt-deploy-action" # We are passing description in the dbt deploy action and using that as search term over here because that's the only search term we have

- name: Validate DBT Deploy Action
id: validate-dbt-deploy-action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: true
dag_tarball_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before-dag-deploy.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after-dag-deploy.outputs.desired_image_version }}
shell: bash
run: |
if [[ "${{ steps.get-dbt-bundle-info.outputs.desired_bundle_version }}" == "" ]]; then
echo "DBT Deploy Action validation failed: DBT bundle not found"
exit 1
fi
if [[ "${{ steps.get-dbt-bundle-info.outputs.bundle_type }}" != "dbt" ]]; then
echo "DBT Deploy Action validation failed: DBT bundle not found"
exit 1
fi
echo "DBT Deploy Action validation succeeded: DBT bundle found"
exit 0
- name: Delete Deployment Preview
id: delete-deployment-preview
Expand Down
29 changes: 14 additions & 15 deletions action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ runs:
dbt_deploy=0
for file in $files; do
if [[ ${{ inputs.root-folder }} == "" || $file == ^"${{ inputs.root-folder }}"* ]]; then
if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
echo $file is part of configured root folder, so would be triggering a dbt deploy
dbt_deploy=1
fi
Expand Down Expand Up @@ -392,7 +392,7 @@ runs:
# 2. If any file is not part of the dags folder, then it triggers a full image build
# 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy
for file in $files; do
if [[ ${{ inputs.root-folder }} == "" || $file == ^"${{ inputs.root-folder }}"* ]]; then
if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
echo $file is part of the input root folder
skip_deploy=0
if [[ $file != *"dags/"* ]]; then
Expand All @@ -403,17 +403,17 @@ runs:
fi
done
# check if user has passed a custom image, then we would need to deploy the image
if [[ ${{ inputs.image-name }} != "no-custom-image" ]]; then
skip_deploy=0
dags_only=0
if [[ $skip_deploy == 1 ]]; then
# skip all deploy steps
dags_only=2
fi
if [[ ${{steps.dag-deploy-enabled.outputs.DAG_DEPLOY_ENABLED}} == false ]]; then
# check if user has passed a custom image, then we would need to deploy the image
if [[ ${{ inputs.image-name }} != "no-custom-image" || ${{ steps.dag-deploy-enabled.outputs.DAG_DEPLOY_ENABLED }} == false || ${{ steps.deployment-preview.outputs.IMAGE_DEPLOY_ONLY }} == true ]]; then
dags_only=0
fi
if [[ ${{steps.deployment-preview.outputs.SKIP_DEPLOY}} == true || $skip_deploy == 1 ]]; then
if [[ ${{ steps.deployment-preview.outputs.SKIP_DEPLOY }} == true ]]; then
# skip all deploy steps
dags_only=2
fi
Expand All @@ -427,7 +427,7 @@ runs:
if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' }}
run: |
echo ::group::Setup Deploy Options
echo ::group::Setup image or dags deploy options
options=""
# add parse option
Expand Down Expand Up @@ -473,6 +473,7 @@ runs:
# only inspect development mode if deployment is not hybrid
if [[ $DEPLOYMENT_TYPE != "HYBRID" ]]; then
DEV_MODE=$(astro deployment inspect ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --clean-output --key configuration.is_development_mode)
echo "Deployment development mode: $DEV_MODE"
if [[ $DEV_MODE == "" ]]; then
echo "DEVELOPMENT_MODE=false" >> $GITHUB_OUTPUT
else
Expand All @@ -488,17 +489,16 @@ runs:
shell: bash
id: development-mode
- name: Override to wake up the Deployment
if: ${{ steps.deployment-preview.outputs.SKIP_DEPLOY == false && steps.development-mode.outputs.DEVELOPMENT_MODE == true }}
if: ${{ steps.deployment-preview.outputs.SKIP_DEPLOY == 'false' && steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }}
run: |
echo ::group::Override to wake up the Deployment
astro deployment wake-up ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --force
# Give it some time to wake up
sleep 60
echo ::endgroup::
shell: bash
- name: DAG Deploy to Astro
if: ${{ inputs.deploy-type == 'dags-only' || (inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == 1) }}
if: ${{ inputs.deploy-type == 'dags-only' || (inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == '1') }}
run: |
echo ::group::DAG Deploy to Astro
Expand All @@ -509,9 +509,8 @@ runs:
shell: bash
# If any other files changed or dag deploys is disabled, deploy the entire Astro project
- name: Image and DAG Deploy to Astro
if: ${{ inputs.deploy-type == 'image-and-dags' || inputs.deploy-image == true || (inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == 0) }}
if: ${{ inputs.deploy-type == 'image-and-dags' || inputs.deploy-image == true || ( inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == '0') }}
run: |
echo ::group::Image and DAG Deploy to Astro
# Deploy image and DAGs
cd ${{ inputs.root-folder }}
Expand Down Expand Up @@ -544,7 +543,7 @@ runs:
echo ::endgroup::
shell: bash
- name: Remove override on Deployment to resume schedule
if: ${{ steps.deployment-preview.outputs.SKIP_DEPLOY == false && steps.development-mode.outputs.DEVELOPMENT_MODE == true }}
if: ${{ steps.deployment-preview.outputs.SKIP_DEPLOY == 'false' && steps.development-mode.outputs.DEVELOPMENT_MODE == 'true' }}
run: |
echo ::group::Remove override on Deployment to resume schedule
Expand Down
File renamed without changes.
4 changes: 4 additions & 0 deletions e2e-setup/mocks/dbt-deploy-git.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/bash

# hack to mock git commands as part of action.yaml so that we could simulate dbt deploy scenario without making any additional commits
echo "e2e-setup/dbt/dbt_project.yml"

0 comments on commit 203064b

Please sign in to comment.