Skip to content

Commit

Permalink
Removes logs and updates workflow configuration to get ready for prod.
Browse files Browse the repository at this point in the history
  • Loading branch information
Andre Giron committed Apr 4, 2023
1 parent 1faecc0 commit 6eacabf
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 116 deletions.
221 changes: 109 additions & 112 deletions .github/workflows/periodic-integration-tests.yml
View file
Original file line number Diff line number Diff line change
@@ -1,105 +1,102 @@
name: Periodic Integration Tests

on:
push:
branches:
- "eng-2639-add-automated-periodic-testing-coverage"
schedule:
- cron: "0 12 * * 1-5" # Run at 4AM PST on every weekday
workflow_dispatch:

jobs:
# run-k8s-tests:
# runs-on: ubuntu-latest-8-cores
# timeout-minutes: 90
# name: SDK Integration Tests against K8s Compute
# steps:
# - uses: actions/checkout@v2
#
# - uses: ./.github/actions/setup-server
# timeout-minutes: 5
#
# # TODO(ENG-2537): Use our separate GH actions credentials.
# - uses: ./.github/actions/fetch-test-config
# with:
# aws_access_key_id: ${{ secrets.KENNY_AWS_ACCESS_KEY_ID }}
# aws_secret_access_key: ${{ secrets.KENNY_AWS_SECRET_ACCESS_KEY }}
# s3_test_config_path: periodic-compute-test-config.yml
#
# - name: Install Terraform
# run: |
# wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
# echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
# sudo apt update && sudo apt install terraform
#
# - name: Create K8s Cluster
# working-directory: integration_tests/sdk/compute/k8s
# run: |
# terraform init
# terraform apply --auto-approve
#
# - name: Update the Kubeconfig file
# working-directory: integration_tests/sdk/compute/k8s
# run: aws eks --region $(terraform output -raw region) update-kubeconfig --name $(terraform output -raw cluster_name)
#
# - name: Update the test-config file with the appropriate cluster name
# working-directory: integration_tests/sdk
# run: sed -i "s/\(cluster_name:\s*\).*/\1 "$(terraform -chdir=compute/k8s/ output -raw cluster_name)"/" test-credentials.yml
#
# - name: Install any data connector packages
# run: |
# aqueduct install s3
# aqueduct install snowflake
#
# - name: Run the SDK Integration Tests
# working-directory: integration_tests/sdk
# run: pytest aqueduct_tests/ -rP -vv -n 4
#
# - name: Teardown K8s Cluster
# id: k8s_cleanup # so that we only upload the state on this specific failure.
# if: always()
# uses: nick-fields/retry@v2
# with:
# max_attempts: 2
# retry_on: error
# timeout_minutes: 20
# retry_wait_seconds: 300
# command: |
# cd integration_tests/sdk/compute/k8s
# terraform destroy --auto-approve
#
# # This directory is quite large, so we only upload it on failure.
# - uses: actions/upload-artifact@v3
# if: ${{ failure() && steps.k8s_cleanup.outcome == 'failure' }}
# with:
# name: Terraform State
# path: |
# integration_tests/sdk/compute/k8s/*
#
# - uses: ./.github/actions/upload-artifacts
# if: always()
# with:
# prefix: K8s Compute
#
# # Sets it as an environmental variable.
# - name: Get the Slack ID for the current oncall
# if: always()
# run: |
# aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
# echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV
#
# - name: Report to Slack on Failure
# if: always()
# uses: ravsamhq/notify-slack-action@v1
# with:
# status: ${{ job.status }}
# notification_title: ""
# message_format: "{emoji} *{workflow}* has {status_message}"
# footer: "{run_url}"
# notify_when: "failure,warnings"
# mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
run-k8s-tests:
runs-on: ubuntu-latest-8-cores
timeout-minutes: 90
name: SDK Integration Tests against K8s Compute
steps:
- uses: actions/checkout@v2

- uses: ./.github/actions/setup-server
timeout-minutes: 5

# TODO(ENG-2537): Use our separate GH actions credentials.
- uses: ./.github/actions/fetch-test-config
with:
aws_access_key_id: ${{ secrets.KENNY_AWS_ACCESS_KEY_ID }}
aws_secret_access_key: ${{ secrets.KENNY_AWS_SECRET_ACCESS_KEY }}
s3_test_config_path: periodic-compute-test-config.yml

- name: Install Terraform
run: |
wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update && sudo apt install terraform
- name: Create K8s Cluster
working-directory: integration_tests/sdk/compute/k8s
run: |
terraform init
terraform apply --auto-approve
- name: Update the Kubeconfig file
working-directory: integration_tests/sdk/compute/k8s
run: aws eks --region $(terraform output -raw region) update-kubeconfig --name $(terraform output -raw cluster_name)

- name: Update the test-config file with the appropriate cluster name
working-directory: integration_tests/sdk
run: sed -i "s/\(cluster_name:\s*\).*/\1 "$(terraform -chdir=compute/k8s/ output -raw cluster_name)"/" test-credentials.yml

- name: Install any data connector packages
run: |
aqueduct install s3
aqueduct install snowflake
- name: Run the SDK Integration Tests
working-directory: integration_tests/sdk
run: pytest aqueduct_tests/ -rP -vv -n 4

- name: Teardown K8s Cluster
id: k8s_cleanup # so that we only upload the state on this specific failure.
if: always()
uses: nick-fields/retry@v2
with:
max_attempts: 2
retry_on: error
timeout_minutes: 20
retry_wait_seconds: 300
command: |
cd integration_tests/sdk/compute/k8s
terraform destroy --auto-approve
# This directory is quite large, so we only upload it on failure.
- uses: actions/upload-artifact@v3
if: ${{ failure() && steps.k8s_cleanup.outcome == 'failure' }}
with:
name: Terraform State
path: |
integration_tests/sdk/compute/k8s/*
- uses: ./.github/actions/upload-artifacts
if: always()
with:
prefix: K8s Compute

# Sets it as an environmental variable.
- name: Get the Slack ID for the current oncall
if: always()
run: |
aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV
- name: Report to Slack on Failure
if: always()
uses: ravsamhq/notify-slack-action@v1
with:
status: ${{ job.status }}
notification_title: ""
message_format: "{emoji} *{workflow}* has {status_message}"
footer: "{run_url}"
notify_when: "failure,warnings"
mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

run-data-integration-tests:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -142,21 +139,21 @@ jobs:
prefix: Data Connectors

# Sets it as an environmental variable.
# - name: Get the Slack ID for the current oncall
# if: always()
# run: |
# aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
# echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV

# - name: Report to Slack on Failure
# if: always()
# uses: ravsamhq/notify-slack-action@v1
# with:
# status: ${{ job.status }}
# notification_title: ""
# message_format: "{emoji} *{workflow}* has {status_message}"
# footer: "{run_url}"
# notify_when: "failure,warnings"
# mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
- name: Get the Slack ID for the current oncall
if: always()
run: |
aws s3 cp s3://aqueduct-assets/oncall.yml ./oncall.yml
echo "ONCALL_SLACK_MEMBER_ID=$(python3 scripts/get_current_oncall.py --file ./oncall.yml)" >> $GITHUB_ENV
- name: Report to Slack on Failure
if: always()
uses: ravsamhq/notify-slack-action@v1
with:
status: ${{ job.status }}
notification_title: ""
message_format: "{emoji} *{workflow}* has {status_message}"
footer: "{run_url}"
notify_when: "failure,warnings"
mention_users: ${{ env.ONCALL_SLACK_MEMBER_ID }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
4 changes: 0 additions & 4 deletions integration_tests/sdk/setup_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,7 @@ def _setup_external_sqlite_db(path: str):


def _setup_postgres_db():
print("Installing Aqueduct Postgres Connector")
_execute_command(["aqueduct", "install", "postgres"])
print("Finished installing Postgres Connector")


def _setup_relational_data(client: Client, db: RelationalDBIntegration) -> None:
Expand Down Expand Up @@ -212,15 +210,13 @@ def setup_data_integrations(client: Client, filter_to: Optional[str] = None) ->
for integration_name in data_integrations:
# Only connect to integrations that don't already exist.
if integration_name not in connected_integrations.keys():
print("Fetching integration credentials ...")
integration_config = _fetch_integration_credentials("data", integration_name)

# Stand up the external integration first.
if integration_config["type"] == ServiceType.SQLITE:
_setup_external_sqlite_db(integration_config["database"])

if integration_config["type"] == ServiceType.POSTGRES:
print("Postgres case hit!!")
_setup_postgres_db()

client.connect_integration(
Expand Down

0 comments on commit 6eacabf

Please sign in to comment.