Skip to content

Commit

Permalink
Code cleanup, re-enables kubernetes tests and Slack notifier
Browse files Browse the repository at this point in the history
  • Loading branch information
Andre Giron committed Mar 24, 2023
1 parent 6f8ee89 commit a4b2f1b
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 114 deletions.
216 changes: 108 additions & 108 deletions .github/workflows/periodic-integration-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,96 +9,96 @@ on:
workflow_dispatch:

jobs:
# run-k8s-tests:
# runs-on: ubuntu-latest-8-cores
# timeout-minutes: 90
# name: SDK Integration Tests against K8s Compute
# steps:
# - uses: actions/checkout@v2

# - uses: ./.github/actions/setup-server
# timeout-minutes: 5

# # TODO(ENG-2537): Use our separate GH actions credentials.
# - uses: ./.github/actions/fetch-test-config
# with:
# aws_access_key_id: ${{ secrets.KENNY_AWS_ACCESS_KEY_ID }}
# aws_secret_access_key: ${{ secrets.KENNY_AWS_SECRET_ACCESS_KEY }}
# s3_test_config_path: periodic-compute-test-config.yml

# - name: Install Terraform
# run: |
# wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
# echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
# sudo apt update && sudo apt install terraform

# - name: Create K8s Cluster
# working-directory: integration_tests/sdk/compute/k8s
# run: |
# terraform init
# terraform apply --auto-approve

# - name: Update the Kubeconfig file
# working-directory: integration_tests/sdk/compute/k8s
# run: aws eks --region $(terraform output -raw region) update-kubeconfig --name $(terraform output -raw cluster_name)

# - name: Update the test-config file with the appropriate cluster name
# working-directory: integration_tests/sdk
# run: sed -i "s/\(cluster_name:\s*\).*/\1 "$(terraform -chdir=compute/k8s/ output -raw cluster_name)"/" test-credentials.yml

# - name: Install any data connector packages
# run: |
# aqueduct install s3
# aqueduct install snowflake

# - name: Run the SDK Integration Tests
# working-directory: integration_tests/sdk
# run: pytest aqueduct_tests/ -rP -vv -n 4

# - name: Teardown K8s Cluster
# if: always()
# uses: nick-fields/retry@v2
# with:
# max_attempts: 2
# retry_on: error
# timeout_minutes: 20
# retry_wait_seconds: 300
# command: |
# cd integration_tests/sdk/compute/k8s
# terraform destroy --auto-approve

# # This directory is quite large, so we only upload it on failure.
# - uses: actions/upload-artifact@v3
# if: ${{ failure() }}
# with:
# name: Terraform State
# path: |
# integration_tests/sdk/compute/k8s/*

# - uses: ./.github/actions/upload-artifacts
# if: always()
# with:
# prefix: K8s Compute

# # Sets it as an environmental variable.
# - name: Get the Slack ID for the current oncall
# if: always()
# run: |
# aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
# echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV

# - name: Report to Slack on Failure
# if: always()
# uses: ravsamhq/notify-slack-action@v1
# with:
# status: ${{ job.status }}
# notification_title: ""
# message_format: "{emoji} *{workflow}* has {status_message}"
# footer: "{run_url}"
# notify_when: "failure,warnings"
# mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
run-k8s-tests:
runs-on: ubuntu-latest-8-cores
timeout-minutes: 90
name: SDK Integration Tests against K8s Compute
steps:
- uses: actions/checkout@v2

- uses: ./.github/actions/setup-server
timeout-minutes: 5

# TODO(ENG-2537): Use our separate GH actions credentials.
- uses: ./.github/actions/fetch-test-config
with:
aws_access_key_id: ${{ secrets.KENNY_AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.KENNY_AWS_SECRET_ACCESS_KEY }}
s3_test_config_path: periodic-compute-test-config.yml

- name: Install Terraform
run: |
wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update && sudo apt install terraform
- name: Create K8s Cluster
working-directory: integration_tests/sdk/compute/k8s
run: |
terraform init
terraform apply --auto-approve
- name: Update the Kubeconfig file
working-directory: integration_tests/sdk/compute/k8s
run: aws eks --region $(terraform output -raw region) update-kubeconfig --name $(terraform output -raw cluster_name)

- name: Update the test-config file with the appropriate cluster name
working-directory: integration_tests/sdk
run: sed -i "s/\(cluster_name:\s*\).*/\1 "$(terraform -chdir=compute/k8s/ output -raw cluster_name)"/" test-credentials.yml

- name: Install any data connector packages
run: |
aqueduct install s3
aqueduct install snowflake
- name: Run the SDK Integration Tests
working-directory: integration_tests/sdk
run: pytest aqueduct_tests/ -rP -vv -n 4

- name: Teardown K8s Cluster
if: always()
uses: nick-fields/retry@v2
with:
max_attempts: 2
retry_on: error
timeout_minutes: 20
retry_wait_seconds: 300
command: |
cd integration_tests/sdk/compute/k8s
terraform destroy --auto-approve
# This directory is quite large, so we only upload it on failure.
- uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: Terraform State
path: |
integration_tests/sdk/compute/k8s/*
- uses: ./.github/actions/upload-artifacts
if: always()
with:
prefix: K8s Compute

# Sets it as an environmental variable.
- name: Get the Slack ID for the current oncall
if: always()
run: |
aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV
- name: Report to Slack on Failure
if: always()
uses: ravsamhq/notify-slack-action@v1
with:
status: ${{ job.status }}
notification_title: ""
message_format: "{emoji} *{workflow}* has {status_message}"
footer: "{run_url}"
notify_when: "failure,warnings"
mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

run-data-integration-tests:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -141,21 +141,21 @@ jobs:
prefix: Data Connectors

# Sets it as an environmental variable.
# - name: Get the Slack ID for the current oncall
# if: always()
# run: |
# aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
# echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV

# - name: Report to Slack on Failure
# if: always()
# uses: ravsamhq/notify-slack-action@v1
# with:
# status: ${{ job.status }}
# notification_title: ""
# message_format: "{emoji} *{workflow}* has {status_message}"
# footer: "{run_url}"
# notify_when: "failure,warnings"
# mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
- name: Get the Slack ID for the current oncall
if: always()
run: |
aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV
- name: Report to Slack on Failure
if: always()
uses: ravsamhq/notify-slack-action@v1
with:
status: ${{ job.status }}
notification_title: ""
message_format: "{emoji} *{workflow}* has {status_message}"
footer: "{run_url}"
notify_when: "failure,warnings"
mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
7 changes: 1 addition & 6 deletions integration_tests/sdk/setup_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,9 @@ def _setup_external_sqlite_db(path: str):


def _setup_postgres_db():
import os
print("Installing Aqueduct Postgres Connector")
_execute_command(["aqueduct", "install", "postgres"])
print("Setting up postgres docker container ...")
# docker run --name aqueduct-postgres -e POSTGRES_PASSWORD=aqueduct -e POSTGRES_DB=aqueducttest -d -p 5432:5432 postgres
# _execute_command(["docker", "run", "--name", "aqueduct-postgres", "-e", "POSTGRES_PASSWORD=aqueduct",
# "-e", "POSTGRES_DB=aqueducttest", "-d", "-p", "5432:5432", "postgres"])
print("Finished setting up Postgres docker container")
print("Finished installing Postgres Connector")


def _setup_relational_data(client: Client, db: RelationalDBIntegration) -> None:
Expand Down

0 comments on commit a4b2f1b

Please sign in to comment.