From 980a1433fbb2b4d268cdac70a83055b82ced339d Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 16 Apr 2021 09:26:51 +0900 Subject: [PATCH 1/5] Test status checks --- .github/workflows/labeler.yml | 9 ++- .github/workflows/notify_test_workflow.yml | 66 ++++++++++++---- .github/workflows/update_build_status.yml | 90 ++++++++++++++++++++++ 3 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/update_build_status.yml diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index a1a5ab5b70f5b..98855f4668b45 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -17,11 +17,18 @@ # under the License. # -name: "Pull Request Labeler" +# Intentionally has a general name. +# because the test status check created in GitHub Actions +# currently randomly picks any associated workflow. +# So, the name was changed to make sense in that context too. +# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10 + +name: "On pull requests" on: pull_request_target jobs: label: + name: Label pull requests runs-on: ubuntu-latest steps: # In order to get back the negated matches like in the old config, diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index 2e459ca8204d1..0e30cd2b5442d 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -1,10 +1,35 @@ -name: Notify test workflow +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Intentionally has a general name. +# because the test status check created in GitHub Actions +# currently randomly picks any associated workflow. +# So, the name was changed to make sense in that context too. +# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10 +name: On pull request update on: pull_request_target: types: [opened, reopened, synchronize] jobs: notify: + name: Notify test workflow runs-on: ubuntu-20.04 steps: - name: "Notify test workflow" @@ -14,6 +39,10 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const endpoint = "GET /repos/:owner/:repo/actions/workflows/:id/runs?&branch=:branch" + + // TODO: Should use pull_request.user and pull_request.user.repos_url? + // If a different person creates a commit to another forked repo, + // it would be able to detect. const params = { owner: context.payload.pull_request.head.repo.owner.login, repo: context.payload.pull_request.head.repo.name, @@ -22,19 +51,28 @@ jobs: } const runs = await github.request(endpoint, params) - var runID = runs.data.workflow_runs[0].id + const runID = runs.data.workflow_runs[0].id + const runUrl = "https://github.com/" + + context.payload.pull_request.head.repo.full_name + + "/actions/runs/" + + runID - var msg = "**[Test build #" + runID + "]" - + "(https://github.com/" + context.payload.pull_request.head.repo.full_name - + "/actions/runs/" + runID + ")** " - + "for PR " + context.issue.number - + " at commit [`" + context.payload.pull_request.head.sha.substring(0, 7) + "`]" - + "(https://github.com/" + context.payload.pull_request.head.repo.full_name - + "/commit/" + context.payload.pull_request.head.sha + ")." + const name = 'Build and test' + const head_sha = context.payload.pull_request.head.sha + const status = 'in_progress' - github.issues.createComment({ - issue_number: context.issue.number, - owner: context.payload.repository.owner.login, - repo: context.payload.repository.name, - body: msg + github.checks.create({ + ...context.repo, + name, + head_sha, + status, + output: { + title: 'Test results', + summary: runUrl, + text: JSON.stringify({ + owner: context.payload.pull_request.head.repo.owner.login, + repo: context.payload.pull_request.head.repo.name, + run_id: runID + }) + } }) diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml new file mode 100644 index 0000000000000..8b6cb73608f23 --- /dev/null +++ b/.github/workflows/update_build_status.yml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Intentionally has a general name. +# because the test status check created in GitHub Actions +# currently randomly picks any associated workflow. +# So, the name was changed to make sense in that context too. +# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10 +name: Scheduled + +on: + schedule: + - cron: "*/15 * * * *" + +jobs: + update: + name: Update build status + runs-on: ubuntu-20.04 + steps: + - name: "Update build status" + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const endpoint = "GET /repos/:owner/:repo/pulls?state=:state" + const params = { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open' + } + + const maybeReady = ["clean", "has_hooks", "unknown", "unstable"]; + const notReady = ["dirty", "draft"]; + + for await (const prs of github.paginate.iterator(endpoint,params)) { + for await (const pr of prs.data) { + if (pr.mergeable_state == null || maybeReady.includes(pr.mergeable_state)) { + const checkRuns = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/check-runs', { + owner: context.repo.owner, + repo: context.repo.repo, + ref: pr.head.sha + }) + + for await (const cr of checkRuns.data.check_runs) { + if (cr.name == "Build and test") { + const params = JSON.parse(cr.output.text) // text contains parameters to make request in JSON + const run = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}', params) + + // Keep syncing the status of the checks + if (run.data.status == "completed") { + const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: cr.id, + output: cr.output, + status: run.data.status, + conclusion: run.data.conclusion + }) + } else { + const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: cr.id, + output: cr.output, + status: run.data.status, + }) + } + + break + } + } + } + } + } From 17430e1e8e9b5a070860117afdd555d490ad56dd Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 16 Apr 2021 19:28:01 +0900 Subject: [PATCH 2/5] Cleanup --- .github/workflows/notify_test_workflow.yml | 23 ++++++++++++----- .github/workflows/update_build_status.yml | 29 +++++++++++++--------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index 0e30cd2b5442d..e9489cb0638b8 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -38,28 +38,39 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const endpoint = "GET /repos/:owner/:repo/actions/workflows/:id/runs?&branch=:branch" + const endpoint = 'GET /repos/:owner/:repo/actions/workflows/:id/runs?&branch=:branch' // TODO: Should use pull_request.user and pull_request.user.repos_url? // If a different person creates a commit to another forked repo, - // it would be able to detect. + // it wouldn't be able to detect. const params = { owner: context.payload.pull_request.head.repo.owner.login, repo: context.payload.pull_request.head.repo.name, - id: "build_and_test.yml", + id: 'build_and_test.yml', branch: context.payload.pull_request.head.ref, } + console.log('SHA: ' + context.payload.pull_request.head.ref) + + // Wait 3 seconds to make sure the fork repository triggered a workflow. + await new Promise(r => setTimeout(r, 3000)) + const runs = await github.request(endpoint, params) const runID = runs.data.workflow_runs[0].id - const runUrl = "https://github.com/" + // TODO: If no workflows were found, it's likely GitHub Actions was not enabled + + if (runs.data.workflow_runs[0].head_sha != context.payload.pull_request.head.sha) { + throw new Error('There was a new unsynced commit pushed. Please retrigger the workflow.'); + } + + const runUrl = 'https://github.com/' + context.payload.pull_request.head.repo.full_name - + "/actions/runs/" + + '/actions/runs/' + runID const name = 'Build and test' const head_sha = context.payload.pull_request.head.sha - const status = 'in_progress' + const status = 'queued' github.checks.create({ ...context.repo, diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml index 8b6cb73608f23..777f147f09b31 100644 --- a/.github/workflows/update_build_status.yml +++ b/.github/workflows/update_build_status.yml @@ -17,12 +17,7 @@ # under the License. # -# Intentionally has a general name. -# because the test status check created in GitHub Actions -# currently randomly picks any associated workflow. -# So, the name was changed to make sense in that context too. -# See also https://github.community/t/specify-check-suite-when-creating-a-checkrun/118380/10 -name: Scheduled +name: Update build status workflow on: schedule: @@ -38,18 +33,22 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const endpoint = "GET /repos/:owner/:repo/pulls?state=:state" + const endpoint = 'GET /repos/:owner/:repo/pulls?state=:state' const params = { owner: context.repo.owner, repo: context.repo.repo, state: 'open' } - const maybeReady = ["clean", "has_hooks", "unknown", "unstable"]; - const notReady = ["dirty", "draft"]; + // See https://docs.github.com/en/graphql/reference/enums#mergestatestatus + const maybeReady = ['behind', 'clean', 'draft', 'has_hooks', 'unknown', 'unstable']; + // Iterate open PRs for await (const prs of github.paginate.iterator(endpoint,params)) { + // Each page for await (const pr of prs.data) { + console.log('SHA: ' + pr.head.sha) + console.log(' Mergeable status: ' + pr.mergeable_state) if (pr.mergeable_state == null || maybeReady.includes(pr.mergeable_state)) { const checkRuns = await github.request('GET /repos/{owner}/{repo}/commits/{ref}/check-runs', { owner: context.repo.owner, @@ -57,13 +56,18 @@ jobs: ref: pr.head.sha }) + // Iterator GitHub Checks in the PR for await (const cr of checkRuns.data.check_runs) { - if (cr.name == "Build and test") { - const params = JSON.parse(cr.output.text) // text contains parameters to make request in JSON + if (cr.name == 'Build and test') { + // text contains parameters to make request in JSON. + const params = JSON.parse(cr.output.text) + + // Get the workflow run in the forked repository const run = await github.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}', params) // Keep syncing the status of the checks - if (run.data.status == "completed") { + if (run.data.status == 'completed') { + console.log(' Run ' + cr.id + ': set status (' + run.data.status + ') and conclusion (' + run.data.conclusion + ')') const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { owner: context.repo.owner, repo: context.repo.repo, @@ -73,6 +77,7 @@ jobs: conclusion: run.data.conclusion }) } else { + console.log(' Run ' + cr.id + ': set status (' + run.data.status + ')') const response = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', { owner: context.repo.owner, repo: context.repo.repo, From 5385deebeb70b2bed85134ab53cacc8da4558c93 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 16 Apr 2021 19:39:27 +0900 Subject: [PATCH 3/5] log ref and sha --- .github/workflows/notify_test_workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index e9489cb0638b8..f1813c9b4c24b 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -50,7 +50,8 @@ jobs: branch: context.payload.pull_request.head.ref, } - console.log('SHA: ' + context.payload.pull_request.head.ref) + console.log('Ref: ' + context.payload.pull_request.head.ref) + console.log('SHA: ' + context.payload.pull_request.head.sha) // Wait 3 seconds to make sure the fork repository triggered a workflow. await new Promise(r => setTimeout(r, 3000)) From 15a8de3d73acc0899dc44c689cf37e01a382ef5f Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 16 Apr 2021 23:16:06 +0800 Subject: [PATCH 4/5] commit1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa7d1dd338be0..4f822a1bec463 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Apache Spark +# Apache Spark Test1 Spark is a unified analytics engine for large-scale data processing. It provides high-level APIs in Scala, Java, Python, and R, and an optimized engine that From 90fb5ea558e4b532ad288b62653fbf935360eacd Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 16 Apr 2021 23:17:18 +0800 Subject: [PATCH 5/5] commit2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f822a1bec463..7489c81873379 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Apache Spark Test1 +# Apache Spark Test1 2 Spark is a unified analytics engine for large-scale data processing. It provides high-level APIs in Scala, Java, Python, and R, and an optimized engine that