From 68deef5b66e6ab1cfdab7a18a6e4d91badca5ade Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Fri, 11 Aug 2023 12:33:39 +0100 Subject: [PATCH 1/4] Try to enable tests from external forks --- .github/workflows/test.yml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68bbaa3d72..6ef0f65475 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,8 @@ on: branches: [main] pull_request: # Run on pull requests targeting the default branch branches: [main] + pull_request_target: # Run on pull requests originated from forks + branches: [main] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -12,14 +14,26 @@ concurrency: jobs: + Authorize: + environment: + ${{ github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + 'external' || 'internal' }} + runs-on: ubuntu-latest + steps: + - run: true + Run-Unit-Tests: runs-on: ubuntu-latest + needs: Authorize strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] airflow-version: ["2.3", "2.4", "2.5", "2.6"] steps: - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -41,6 +55,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} Run-Integration-Tests: + needs: Authorize runs-on: ubuntu-latest strategy: matrix: @@ -62,6 +77,8 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -94,30 +111,16 @@ jobs: DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} Run-Integration-Tests-Expensive: + needs: Authorize runs-on: ubuntu-latest strategy: matrix: python-version: ["3.10"] airflow-version: ["2.6"] - if: >- - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false - ) || - ( - github.event_name == 'pull_request_target' && - contains(github.event.pull_request.labels.*.name, 'safe') - ) steps: - uses: actions/checkout@v3 - if: github.event_name != 'pull_request_target' - - - name: Checkout pull/${{ github.event.number }} - uses: actions/checkout@v3 with: - ref: ${{ github.event.pull_request.head.sha }} - if: github.event_name == 'pull_request_target' + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 192bef8f67c28f1157c0a5afe5191bff8083cf97 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 14 Aug 2023 10:28:54 +0100 Subject: [PATCH 2/4] Try to fix CI env retrieval for forks --- .github/workflows/test.yml | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ef0f65475..51fee91ee3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,6 @@ jobs: Run-Unit-Tests: runs-on: ubuntu-latest - needs: Authorize strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] @@ -94,21 +93,22 @@ jobs: run: | hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + Run-Integration-Tests-Expensive: needs: Authorize @@ -131,6 +131,15 @@ jobs: run: | python -m pip install hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} run: | @@ -142,12 +151,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + From ea55d95e543f9c9d3a5fd4c92cefa5a8aefc6d46 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 14 Aug 2023 10:50:11 +0100 Subject: [PATCH 3/4] Add COSMOS_CONN_POSTGRES_PASSWORD secret --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51fee91ee3..75697eab1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,7 @@ jobs: DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -140,6 +141,7 @@ jobs: DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} run: | @@ -150,5 +152,3 @@ jobs: uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - - From 12136dd3db4ff9cbef801ab77fd981ce84dea2b6 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 14 Aug 2023 11:14:11 +0100 Subject: [PATCH 4/4] Fix envvar for expensive integration tests --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75697eab1b..8d65e0300a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -132,6 +132,11 @@ jobs: run: | python -m pip install hatch hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup + DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres @@ -143,11 +148,6 @@ jobs: DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} - run: | - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup - DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive - - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: