diff --git a/.github/workflows/copy-workflow.yml b/.github/workflows/copy-workflow.yml index af0f407e..21940fc2 100644 --- a/.github/workflows/copy-workflow.yml +++ b/.github/workflows/copy-workflow.yml @@ -5,9 +5,6 @@ name: Deploy on: workflow_dispatch: inputs: - head_commit_url: - description: "github.event.head_commit.url of the dispatcher" - required: true targets: description: "List of repositories to deploy to" required: true @@ -19,7 +16,7 @@ jobs: fail-fast: false matrix: cfg: ${{ fromJson(github.event.inputs.targets) }} - max-parallel: 1 + max-parallel: 10 env: TARGET_REPO_DIR: "target-repo" TEMPLATE_REPO_DIR: "template-repo" @@ -28,9 +25,6 @@ jobs: INITIAL_WORKFLOW_DEPLOYMENT: 0 GO_VERSION_BUMP: 0 FILES: "" - GITHUB_USER: "web3-bot" - GITHUB_EMAIL: "web3-bot@users.noreply.github.com" - DEFAULTBRANCH: "" name: ${{ matrix.cfg.target }} steps: - name: Checkout ${{ matrix.cfg.target }} @@ -45,16 +39,11 @@ jobs: uses: actions/checkout@v2 with: path: ${{ env.TEMPLATE_REPO_DIR }} - - name: determine GitHub default branch - working-directory: ${{ env.TARGET_REPO_DIR }} - run: | - defaultbranch=$(git remote show origin | awk '/HEAD branch/ {print $NF}') - echo "DEFAULTBRANCH=$defaultbranch" >> $GITHUB_ENV - name: git config working-directory: ${{ env.TARGET_REPO_DIR }} run: | - git config user.name ${{ env.GITHUB_USER }} - git config user.email ${{ env.GITHUB_EMAIL }} + git config user.name web3-bot + git config user.email web3-bot@users.noreply.github.com - name: determine files to add # By setting the environment variable, it's possible to programmatically add / modify this list. # See https://github.com/protocol/.github/blob/38135c75e47839623bf9b2748275d8c6167a8fa8/.github/workflows/copy-workflow.yml#L163-L168 for an example, how we used to make use of this. @@ -111,16 +100,13 @@ jobs: # add DO NOT EDIT header tmp=$(mktemp) cat $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/header.yml $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f > $tmp - # replace $default-branch with this repo's GitHub default branch - sed -i "s:\$default-branch:${{ env.DEFAULTBRANCH }}:g" $tmp - mv $tmp $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f # create commit, if necessary commit_msg="" if [[ ! -f "$TARGET_REPO_DIR/$f" ]]; then echo "First deployment.\n" commit_msg="add $f" else - status=$(cmp --silent $TARGET_REPO_DIR/$f $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f; echo $?) + status=$(cmp --silent $TARGET_REPO_DIR/$f $tmp; echo $?) if [[ $status -ne 0 ]]; then echo "Update needed." commit_msg="update $f" @@ -129,26 +115,19 @@ jobs: continue fi fi - dir="$TARGET_REPO_DIR/"$(dirname $f) - mkdir -p $dir - cp $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f $dir + mkdir -p "$TARGET_REPO_DIR/$(dirname $f)" + mv $tmp $TARGET_REPO_DIR/$f pushd $TARGET_REPO_DIR > /dev/null git add $f git commit -m "$commit_msg" popd > /dev/null done - - name: Check if we need to create a PR + - name: Check if we need to update the branch working-directory: ${{ env.TARGET_REPO_DIR }} run: echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count 2> /dev/null || echo 1)" >> $GITHUB_ENV - - name: Create Pull Request + - name: Force push web3-bot/sync branch if: ${{ env.NEEDS_UPDATE != 0 }} - uses: peter-evans/create-pull-request@83dbed188f76ab04433c639ec214df65e26bc15c # https://github.com/peter-evans/create-pull-request/pull/856 - with: - path: ${{ env.TARGET_REPO_DIR }} - title: "sync: update CI config files" - body: Syncing to commit ${{ github.event.inputs.head_commit_url }}. - token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} - committer: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}> - author: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}> - branch: ${{ env.GITHUB_USER }}/sync - delete-branch: true + working-directory: ${{ env.TARGET_REPO_DIR }} + run: | + git checkout -B web3-bot/sync + git push origin web3-bot/sync -f diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 59c0eba0..4e7a541d 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -1,6 +1,6 @@ -# Trigger the execution of copy-workflow.yml, in batches of 8 repositories. +# Trigger the execution of copy workflow in batches. # This workflow is needed since GitHub Actions limits the matrix size to 256 jobs. -# We use one job per repository per file. +# We use one job per repository per batch. on: push: @@ -8,14 +8,20 @@ on: env: # Number of repositories in a batch. - # Deployment jobs in the same batch are run sequentially. - # With ~180 repos, this means we'll run around 9 instances of the copy workflow in parallel. - # This is (hopefully) sufficient to prevent us from triggering GitHub Action's abuse detection mechanism. - MAX_REPOS_PER_WORKFLOW: 20 + # Matrix jobs within a copy workflow run are run in parallel. + # 256 is the upper limit on the number of matrix jobs. + # Batching too many repositories together can result in + # could not create workflow dispatch event: HTTP 422: inputs are too large. + # This value should be higher than max-parallel in copy workflow. + MAX_REPOS_PER_WORKFLOW: 100 + # Number of seconds to wait before starting to watch copy workflow run. + # Unfortunately, the interval on the watch is not configurable. + # The delay helps us save on GH API requests. + WORKFLOW_COMPLETION_CHECK_DELAY: 60 jobs: matrix: - name: Trigger copy workflows + name: Batch targets runs-on: ubuntu-latest outputs: batches: ${{ steps.set-matrix.outputs.batches }} @@ -42,6 +48,7 @@ jobs: echo "::set-output name=batches::$batches" dispatch: needs: [ matrix ] + name: Dispatch copy workflow(batch ${{ matrix.cfg.key }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -51,18 +58,87 @@ jobs: # For each "dispatch" job, matrix.cfg.value is an array, like: # # [ - # {"target": "repo1", "files": [".github/workflows/go-check.yml"]}, + # {"target": "repo1", "files": [".github/workflows/go-check.yml"]}, # {"target": "repo2", "files": [".github/workflows/go-check.yml", ".github/workflows/go-test.yml"]} # ] # - # The triggered copy-workflow jobs use that final array as their matrix. - # As such, we'll end up with one copy-workflow parallel job per target. + # The triggered copy workflow runs use that final array as their matrix. + # Since max-parallel here is 1, we'll end up with at most max-parallel from copy workflow + 1 parallel jobs. + # 20 is the upper limit on parallel jobs on a free plan. cfg: ${{ fromJson(needs.matrix.outputs.batches) }} - name: Start copy workflow (batch ${{ matrix.cfg.key }}) + max-parallel: 1 + env: + GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} + WORKFLOW_YML: copy-workflow.yml + WORKFLOW_REPO: protocol/.github steps: - - uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1.1.0 - with: - workflow: "Deploy" # "name" attribute of copy-workflow.yml - token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} - # double toJson on matrix.cfg.value is here on purpose - inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }' + - id: dispatch + name: Dispatch copy workflow + run: | + start_date="$(date +%s)" + # 2 toJson calls are needed to turn the array of targets into a string + echo '{"targets":${{ toJson(toJson(matrix.cfg.value)) }}}' | jq -c '.' | gh workflow run "${{ env.WORKFLOW_YML }}" --ref "${{ github.ref }}" --repo "${{ env.WORKFLOW_REPO }}" --json + echo "::set-output name=start_date::$start_date" + - id: run + name: Wait for copy workflow run to start + run: | + # checks every 3 seconds until the most recent copy workflow run's created_at is later than this job's start_date + while sleep 3; do + run="$(gh api "/repos/${{ env.WORKFLOW_REPO }}/actions/workflows/${{ env.WORKFLOW_YML }}/runs?per_page=1" --jq '.workflow_runs[0]')" + # nothing to check if no copy workflow run was returned + if [[ ! -z "$run" ]]; then + run_start_date="$(date --date="$(jq -r '.created_at' <<< "$run")" +%s)" + if [[ "$run_start_date" > "${{ steps.dispatch.outputs.start_date }}" ]]; then + echo "::set-output name=id::$(jq -r '.id' <<< "$run")" + break + fi + fi + done + - name: Wait for copy workflow run to complete + run: | + # delays checking copy workflow's run status to save on GH API requests + sleep ${{ env.WORKFLOW_COMPLETION_CHECK_DELAY }} + + # checks every 3 seconds until the copy workflow run's status is completed + # redirects the stdout to /dev/null because it is very chatty + gh run watch "${{ steps.run.outputs.id }}" --repo "${{ env.WORKFLOW_REPO }}" > /dev/null + - id: jobs + name: Find copy workflow jobs that pushed changes + env: + STEP_NAME: 'Force push web3-bot/sync branch' + run: | + targets="$(gh api --paginate '/repos/${{ env.WORKFLOW_REPO }}/actions/runs/${{ steps.run.outputs.id }}/jobs' | jq '.jobs | map(select(.steps[] | select(.name == "${{ env.STEP_NAME }}") and select(.conclusion == "success"))) | map(.name)' | jq -sc 'add')" + echo "::set-output name=targets::$targets" + - name: Sync PRs in targets that need it + env: + PR_TITLE: 'sync: update CI config files' + PR_BODY: 'Syncing to commit ${{ github.event.head_commit.url }}.' + PR_BRANCH: 'web3-bot/sync' + run: | + failed=() + for target in $(jq -r '.[]' <<< ${{ toJson(steps.jobs.outputs.targets) }}); do + base="$(gh api "/repos/${target}" --jq '.default_branch')" + # tries to create a PR in target + if result="$(gh api "/repos/$target/pulls" -f title="${{ env.PR_TITLE }}" -f body="${{ env.PR_BODY }}" -f head="${{ env.PR_BRANCH }}" -f base="$base")"; then + echo "Successfully created a PR for '$target'" + elif [[ "$(jq -r '.errors[0].message' <<< "$result")" == 'A pull request already exists'* ]]; then + number="$(gh api "/repos/$target/pulls?head=${{ env.PR_BRANCH }}&per_page=1&state=open" --jq '.[0].number')" + # tries to update the PR in target + if result="$(gh api -X PATCH "/repos/$target/pulls/$number" -f body="${{ env.PR_BODY }}" -f base="$base")"; then + echo "Successfully updated the PR for '$target'" + else + echo "$result" + echo "Failed to update the PR for '$target'" + failed+=("$target(update)") + fi + else + echo "$result" + echo "Failed to create a PR for '$target'" + failed+=("$target(create)") + fi + sleep 3 + done + if ((${#failed[@]})); then + echo "::error ::Failed to sync PRs in: ${failed[@]}" + exit 1 + fi