open PRs sequentially but continue updating branches in parallel

.github/workflows/copy-workflow.yml

@@ -5,9 +5,6 @@ name: Deploy
 on:
   workflow_dispatch:
     inputs:
-      head_commit_url:
-        description: "github.event.head_commit.url of the dispatcher"
-        required: true
       targets:
         description: "List of repositories to deploy to"
         required: true
@@ -19,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         cfg: ${{ fromJson(github.event.inputs.targets) }}
-      max-parallel: 1
+      max-parallel: 10
     env:
       TARGET_REPO_DIR: "target-repo"
       TEMPLATE_REPO_DIR: "template-repo"
@@ -30,7 +27,6 @@ jobs:
       FILES: ""
       GITHUB_USER: "web3-bot"
       GITHUB_EMAIL: "web3-bot@users.noreply.github.com"
-      DEFAULTBRANCH: ""
     name: ${{ matrix.cfg.target }}
     steps:
     - name: Checkout ${{ matrix.cfg.target }}
@@ -45,11 +41,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: ${{ env.TEMPLATE_REPO_DIR }}
-    - name: determine GitHub default branch
-      working-directory: ${{ env.TARGET_REPO_DIR }}
-      run: |
-        defaultbranch=$(git remote show origin | awk '/HEAD branch/ {print $NF}')
-        echo "DEFAULTBRANCH=$defaultbranch" >> $GITHUB_ENV
     - name: git config
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
@@ -111,16 +102,13 @@ jobs:
           # add DO NOT EDIT header
           tmp=$(mktemp)
           cat $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/header.yml $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f > $tmp
-          # replace $default-branch with this repo's GitHub default branch
-          sed -i "s:\$default-branch:${{ env.DEFAULTBRANCH }}:g" $tmp
-          mv $tmp $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f
           # create commit, if necessary
           commit_msg=""
           if [[ ! -f "$TARGET_REPO_DIR/$f" ]]; then
             echo "First deployment.\n"
             commit_msg="add $f"
           else
-            status=$(cmp --silent $TARGET_REPO_DIR/$f $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f; echo $?)
+            status=$(cmp --silent $TARGET_REPO_DIR/$f $tmp; echo $?)
             if [[ $status -ne 0 ]]; then
               echo "Update needed."
               commit_msg="update $f"
@@ -129,26 +117,19 @@ jobs:
               continue
             fi
           fi
-          dir="$TARGET_REPO_DIR/"$(dirname $f)
-          mkdir -p $dir
-          cp $TEMPLATE_REPO_DIR/$TEMPLATE_DIR/$f $dir
+          mkdir -p "$TARGET_REPO_DIR/$(dirname $f)"
+          mv $tmp $TARGET_REPO_DIR/$f
           pushd $TARGET_REPO_DIR > /dev/null
           git add $f
           git commit -m "$commit_msg"
           popd > /dev/null
         done
-    - name: Check if we need to create a PR
+    - name: Check if we need to update the branch
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count 2> /dev/null || echo 1)" >> $GITHUB_ENV
-    - name: Create Pull Request
+    - name: Force push ${{ env.GITHUB_USER }}/sync branch
       if: ${{ env.NEEDS_UPDATE != 0 }}
-      uses: peter-evans/create-pull-request@83dbed188f76ab04433c639ec214df65e26bc15c # https://github.com/peter-evans/create-pull-request/pull/856
-      with:
-        path: ${{ env.TARGET_REPO_DIR }}
-        title: "sync: update CI config files"
-        body: Syncing to commit ${{ github.event.inputs.head_commit_url }}.
-        token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
-        committer: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}>
-        author: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}>
-        branch: ${{ env.GITHUB_USER }}/sync
-        delete-branch: true
+      working-directory: ${{ env.TARGET_REPO_DIR }}
+      run: |
+        git checkout -B ${{ env.GITHUB_USER }}/sync
+        git push origin ${{ env.GITHUB_USER }}/sync -f

.github/workflows/dispatch.yml

@@ -1,21 +1,27 @@
-# Trigger the execution of copy-workflow.yml, in batches of 8 repositories.
+# Trigger the execution of copy workflow in batches.
 # This workflow is needed since GitHub Actions limits the matrix size to 256 jobs.
-# We use one job per repository per file.
+# We use one job per repository per batch.
 
 on:
   push:
     branches: [ master, testing ]
 
 env:
   # Number of repositories in a batch.
-  # Deployment jobs in the same batch are run sequentially.
-  # With ~180 repos, this means we'll run around 9 instances of the copy workflow in parallel.
-  # This is (hopefully) sufficient to prevent us from triggering GitHub Action's abuse detection mechanism.
-  MAX_REPOS_PER_WORKFLOW: 20
+  # Matrix jobs within a copy workflow run are run in parallel.
+  # 256 is the upper limit on the number of matrix jobs.
+  # Batching too many repositories together can result in
+  #  could not create workflow dispatch event: HTTP 422: inputs are too large.
+  # This value should be higher than max-parallel in copy workflow.
+  MAX_REPOS_PER_WORKFLOW: 100
+  # Number of seconds to wait before starting to watch copy workflow run.
+  # Unfortunately, the interval on the watch is not configurable.
+  # The delay helps us save on GH API requests.
+  WORKFLOW_COMPLETION_CHECK_DELAY: 60
 
 jobs:
   matrix:
-    name: Trigger copy workflows
+    name: Batch targets
     runs-on: ubuntu-latest
     outputs:
       batches: ${{ steps.set-matrix.outputs.batches }}
@@ -42,6 +48,7 @@ jobs:
           echo "::set-output name=batches::$batches"
   dispatch:
     needs: [ matrix ]
+    name: Dispatch copy workflow(batch ${{ matrix.cfg.key }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -51,18 +58,87 @@ jobs:
         # For each "dispatch" job, matrix.cfg.value is an array, like:
         #
         #   [
-        #     {"target": "repo1", "files": [".github/workflows/go-check.yml"]}, 
+        #     {"target": "repo1", "files": [".github/workflows/go-check.yml"]},
         #     {"target": "repo2", "files": [".github/workflows/go-check.yml", ".github/workflows/go-test.yml"]}
         #   ]
         #
-        # The triggered copy-workflow jobs use that final array as their matrix.
-        # As such, we'll end up with one copy-workflow parallel job per target.
+        # The triggered copy workflow runs use that final array as their matrix.
+        # Since max-parallel here is 1, we'll end up with at most max-parallel from copy workflow + 1 parallel jobs.
+        # 20 is the upper limit on parallel jobs on a free plan.
         cfg: ${{ fromJson(needs.matrix.outputs.batches) }}
-    name: Start copy workflow (batch ${{ matrix.cfg.key }})
+      max-parallel: 1
+    env:
+      GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
+      WORKFLOW_YML: copy-workflow.yml
+      WORKFLOW_REPO: protocol/.github
     steps:
-      - uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1.1.0
-        with:
-          workflow: "Deploy" # "name" attribute of copy-workflow.yml
-          token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
-          # double toJson on matrix.cfg.value is here on purpose
-          inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }'
+      - id: dispatch
+        name: Dispatch copy workflow
+        run: |
+          start_date="$(date +%s)"
+          # 2 toJson calls are needed to turn the array of targets into a string
+          echo '{"targets":${{ toJson(toJson(matrix.cfg.value)) }}}' | jq -c '.' | gh workflow run "${{ env.WORKFLOW_YML }}" --ref "${{ github.ref }}" --repo "${{ env.WORKFLOW_REPO }}" --json
+          echo "::set-output name=start_date::$start_date"
+      - id: run
+        name: Wait for copy workflow run to start
+        run: |
+          # checks every 3 seconds until the most recent copy workflow run's created_at is later than this job's start_date
+          while sleep 3; do
+            run="$(gh api "/repos/${{ env.WORKFLOW_REPO }}/actions/workflows/${{ env.WORKFLOW_YML }}/runs?per_page=1" --jq '.workflow_runs[0]')"
+            # nothing to check if no copy workflow run was returned
+            if [[ ! -z "$run" ]]; then
+              run_start_date="$(date --date="$(jq -r '.created_at' <<< "$run")" +%s)"
+              if [[ "$run_start_date" > "${{ steps.dispatch.outputs.start_date }}" ]]; then
+                echo "::set-output name=id::$(jq -r '.id' <<< "$run")"
+                break
+              fi
+            fi
+          done
+      - name: Wait for copy workflow run to complete
+        run: |
+          # delays checking copy workflow's run status to save on GH API requests
+          sleep ${{ env.WORKFLOW_COMPLETION_CHECK_DELAY }}
+
+          # checks every 3 seconds until the copy workflow run's status is completed
+          # redirects the stdout to /dev/null because it is very chatty
+          gh run watch "${{ steps.run.outputs.id }}" --repo "${{ env.WORKFLOW_REPO }}" > /dev/null
+      - id: jobs
+        name: Find copy workflow jobs that pushed changes
+        env:
+          STEP_NAME: 'Force push web3-bot/sync branch'
+        run: |
+          targets="$(gh api --paginate '/repos/${{ env.WORKFLOW_REPO }}/actions/runs/${{ steps.run.outputs.id }}/jobs' | jq '.jobs | map(select(.steps[] | select(.name == "${{ env.STEP_NAME }}") and select(.conclusion == "success"))) | map(.name)' | jq -sc 'add')"
+          echo "::set-output name=targets::$targets"
+      - name: Sync PRs in targets that need it
+        env:
+          PR_TITLE: 'sync: update CI config files'
+          PR_BODY: 'Syncing to commit ${{ github.event.head_commit.url }}.'
+          PR_BRANCH: 'web3-bot/sync'
+        run: |
+          failed=()
+          for target in $(jq -r '.[]' <<< ${{ toJson(steps.jobs.outputs.targets) }}); do
+            base="$(gh api "/repos/${target}" --jq '.default_branch')"
+            # tries to create a PR in target
+            if result="$(gh api "/repos/$target/pulls" -f title="${{ env.PR_TITLE }}" -f body="${{ env.PR_BODY }}" -f head="${{ env.PR_BRANCH }}" -f base="$base")"; then
+              echo "Successfully created a PR for '$target'"
+            elif [[ "$(jq -r '.errors[0].message' <<< "$result")" == 'A pull request already exists'* ]]; then
+              number="$(gh api "/repos/$target/pulls?head=${{ env.PR_BRANCH }}&per_page=1&state=open" --jq '.[0].number')"
+              # tries to update the PR in target
+              if result="$(gh api -X PATCH "/repos/$target/pulls/$number" -f body="${{ env.PR_BODY }}" -f base="$base")"; then
+                echo "Successfully updated the PR for '$target'"
+              else
+                echo "$result"
+                echo "Failed to update the PR for '$target'"
+                failed+=("$target(update)")
+              fi
+            else
+              echo "$result"
+              echo "Failed to create a PR for '$target'"
+              failed+=("$target(create)")
+            fi
+            sleep 3
+          done
+          if ((${#failed[@]})); then
+            echo "::error ::Failed to sync PRs in: ${failed[@]}"
+            exit 1
+          fi