diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68bbaa3d72..8d65e0300a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,8 @@ on: branches: [main] pull_request: # Run on pull requests targeting the default branch branches: [main] + pull_request_target: # Run on pull requests originated from forks + branches: [main] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -12,6 +14,15 @@ concurrency: jobs: + Authorize: + environment: + ${{ github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository && + 'external' || 'internal' }} + runs-on: ubuntu-latest + steps: + - run: true + Run-Unit-Tests: runs-on: ubuntu-latest strategy: @@ -20,6 +31,8 @@ jobs: airflow-version: ["2.3", "2.4", "2.5", "2.6"] steps: - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -41,6 +54,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} Run-Integration-Tests: + needs: Authorize runs-on: ubuntu-latest strategy: matrix: @@ -62,6 +76,8 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -77,47 +93,35 @@ jobs: run: | hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + Run-Integration-Tests-Expensive: + needs: Authorize runs-on: ubuntu-latest strategy: matrix: python-version: ["3.10"] airflow-version: ["2.6"] - if: >- - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false - ) || - ( - github.event_name == 'pull_request_target' && - contains(github.event.pull_request.labels.*.name, 'safe') - ) steps: - uses: actions/checkout@v3 - if: github.event_name != 'pull_request_target' - - - name: Checkout pull/${{ github.event.number }} - uses: actions/checkout@v3 with: - ref: ${{ github.event.pull_request.head.sha }} - if: github.event_name == 'pull_request_target' + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -133,18 +137,18 @@ jobs: run: | hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} - DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}