From 6a86ba7eeb32504c9d15ae6aa27a5d550728ee96 Mon Sep 17 00:00:00 2001 From: Elias Segundo Date: Mon, 19 Dec 2022 10:32:17 -0600 Subject: [PATCH 1/3] Adding Setup python to Dataflow job --- .github/workflows/java_tests.yml | 63 ++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml index 1a587e7b4919..d19a38e6c429 100644 --- a/.github/workflows/java_tests.yml +++ b/.github/workflows/java_tests.yml @@ -18,38 +18,37 @@ # To learn more about GitHub Actions in Apache Beam check the CI.md name: Java Tests - on: workflow_dispatch: inputs: runDataflow: description: 'Type "true" if you want to run Dataflow tests (GCP variables must be configured, check CI.md)' - default: false + default: 'false' + required: false schedule: - cron: '10 2 * * *' push: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] pull_request: branches: ['master', 'release-*'] - tags: 'v*' + tags: ['v*'] paths: ['sdks/java/**', 'model/**', 'runners/**', 'examples/java/**', 'examples/kotlin/**', 'release/**', 'buildSrc/**'] - # This allows a subsequently queued workflow run to interrupt previous runs concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true - jobs: check_gcp_variables: timeout-minutes: 5 name: "Check GCP variables set" - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04] outputs: gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }} steps: - - uses: actions/checkout@v3 + - name: Check out code + uses: actions/checkout@v3 - name: "Check are GCP variables set" run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables @@ -67,14 +66,24 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [[self-hosted, ubuntu-20.04], macos-latest, [self-hosted, windows-server-2019]] steps: - - name: Checkout code + - name: Check out code uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - # :sdks:java:core:test + - name: Install Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # net.linguica.gradle.maven.settings plugin and github's provided maven + # settings.xml file + run: rm ~/.m2/settings.xml + # :sdks:java:core:test - name: Run :sdks:java:core:test uses: ./.github/actions/gradle-command-action with: @@ -116,13 +125,23 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [[self-hosted, ubuntu-20.04], macos-latest, [self-hosted, windows-server-2019]] steps: - - name: Checkout code + - name: Check out code uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive + - name: Install Java + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '8' + - name: Remove default github maven configuration + # This step is a workaround to avoid a decryption issue of Beam's + # net.linguica.gradle.maven.settings plugin and github's provided maven + # settings.xml file + run: rm ~/.m2/settings.xml - name: Install go uses: actions/setup-go@v3 with: @@ -145,17 +164,17 @@ jobs: needs: - check_gcp_variables runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [[self-hosted, Windows]] if: | needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && ( (github.event_name == 'push' || github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && github.event.inputs.runDataflow == 'true') ) - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] steps: - - name: Checkout code + - name: Check out code uses: actions/checkout@v3 with: persist-credentials: false @@ -164,6 +183,10 @@ jobs: uses: actions/setup-go@v3 with: go-version: '1.16.12' + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.7' - name: Authenticate on GCP uses: google-github-actions/setup-gcloud@v0 with: @@ -175,7 +198,7 @@ jobs: uses: actions/setup-java@v3.6.0 with: distribution: 'zulu' - java-version: 8 + java-version: '8' - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven @@ -192,4 +215,4 @@ jobs: if: always() with: name: java_wordcount_dataflow-${{matrix.os}} - path: examples/java/build/reports/tests/integrationTest + path: examples/java/build/reports/tests/integrationTest \ No newline at end of file From 600c1431c60719bc48ffdf838b147fd0e945a213 Mon Sep 17 00:00:00 2001 From: Elias Segundo Date: Mon, 19 Dec 2022 10:34:35 -0600 Subject: [PATCH 2/3] Adding Setup python to Dataflow job --- .github/workflows/java_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml index d19a38e6c429..74dfa19c5768 100644 --- a/.github/workflows/java_tests.yml +++ b/.github/workflows/java_tests.yml @@ -167,7 +167,7 @@ jobs: strategy: fail-fast: false matrix: - os: [[self-hosted, Windows]] + os: [[self-hosted, ubuntu-20.04],[self-hosted, windows-server-2019]] if: | needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && ( (github.event_name == 'push' || github.event_name == 'schedule') || From 273d92611da741ba9a3385d20d63c409bdd51f6b Mon Sep 17 00:00:00 2001 From: Elias Segundo Date: Tue, 20 Dec 2022 21:59:56 -0600 Subject: [PATCH 3/3] Using standard setup-self-hosted composite action for installations --- .github/workflows/java_tests.yml | 40 ++++++++++++-------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml index 74dfa19c5768..025ff691d787 100644 --- a/.github/workflows/java_tests.yml +++ b/.github/workflows/java_tests.yml @@ -73,11 +73,11 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Install Java - uses: actions/setup-java@v3 + - name: Setup self-hosted + uses: ./.github/actions/setup-self-hosted-action with: - distribution: 'temurin' - java-version: '8' + requires-py-38: false + requires-py-39: false - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # net.linguica.gradle.maven.settings plugin and github's provided maven @@ -132,20 +132,17 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Install Java - uses: actions/setup-java@v2 + - name: Setup self-hosted + uses: ./.github/actions/setup-self-hosted-action with: - distribution: 'temurin' - java-version: '8' + requires-py-38: false + requires-py-39: false + - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # net.linguica.gradle.maven.settings plugin and github's provided maven # settings.xml file run: rm ~/.m2/settings.xml - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: '1.16.12' - name: Run WordCount Unix uses: ./.github/actions/gradle-command-action with: @@ -179,14 +176,11 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Install go - uses: actions/setup-go@v3 - with: - go-version: '1.16.12' - - name: Install Python - uses: actions/setup-python@v3 - with: - python-version: '3.7' + - name: Setup self-hosted + uses: ./.github/actions/setup-self-hosted-action + with: + requires-py-38: false + requires-py-39: false - name: Authenticate on GCP uses: google-github-actions/setup-gcloud@v0 with: @@ -194,11 +188,7 @@ jobs: service_account_key: ${{ secrets.GCP_SA_KEY }} project_id: ${{ secrets.GCP_PROJECT_ID }} export_default_credentials: true - - name: Set Java Version - uses: actions/setup-java@v3.6.0 - with: - distribution: 'zulu' - java-version: '8' + - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # gradle-command-action plugin and github's provided maven