Prepare copy-workflow for setting up unified CI for JS repos

.githooks/pre-commit

@@ -1,12 +1,15 @@
 #!/bin/bash
 
-tmp=$(mktemp)
-git show :config.json > $tmp
-status=0
-if ! output=$(.github/workflows/check-config.sh $tmp); then
-  echo "$output"
-  status=1
-fi
-rm $tmp
+for config in configs/*.json; do
+  tmp=$(mktemp)
+  git show :$config > $tmp
+  status=0
+  if ! output=$(.github/workflows/check-config.sh $tmp); then
+    echo "$tmp contains git show :$config output."
+    echo "$output"
+    status=1
+  fi
+  rm $tmp
+done
 
 exit $status

.github/actions/copy-workflow-go/action.yml

@@ -0,0 +1,64 @@
+name: copy workflow go
+description: Copy workflow steps specific to go
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-go@v2
+      with:
+        # This should be the same Go version we use in the go-check workflow.
+        # go mod tidy, go vet, staticcheck and gofmt might behave differently depending on the version.
+        go-version: "1.17.x"
+    - name: bump go.mod go version if needed
+      uses: protocol/multiple-go-modules@v1.2
+      with:
+        working-directory: ${{ env.TARGET_REPO_DIR }}
+        run: |
+          # We want our modules to support two Go versions at a time.
+          # As of August 2021, Go 1.17 is the latest stable.
+          # go.mod's Go version declares the language version being used.
+          # As such, it has to be the minimum of all Go versions supported.
+          # Bump this every six months, as new Go versions come out.
+          TARGET_VERSION=1.16
+
+          # Note that the "<" comparison doesn't understand semver,
+          # but it should be good enough for the foreseeable future.
+          CURRENT_VERSION=$(go list -m -f {{.GoVersion}})
+
+          if [[ $CURRENT_VERSION < $TARGET_VERSION ]]; then
+            echo "GO_VERSION_BUMP=1" >> $GITHUB_ENV
+
+            # Update the version in go.mod. This alone ensures there's a diff.
+            go mod edit -go $TARGET_VERSION
+
+            # In the future, "go fix" may make changes to Go code,
+            # such as to adapt to language changes or API deprecations.
+            # This is largely a no-op as of Go 1.17, and that's fine.
+            go fix ./...
+            git add .
+
+            # We don't tidy, because the next step does that.
+            # Separate commits also help with reviews.
+            git commit -m "bump go.mod to Go $TARGET_VERSION and run go fix"
+          fi
+    - name: go mod tidy (on initial workflow deployment and on new Go version)
+      if: ${{ env.INITIAL_WORKFLOW_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
+      uses: protocol/multiple-go-modules@v1.2
+      with:
+        working-directory: ${{ env.TARGET_REPO_DIR }}
+        run: |
+          go mod tidy
+          if ! git diff --quiet; then
+            git add .
+            git commit -m "run go mod tidy"
+          fi
+    - name: gofmt -s (on initial workflow deployment and on new Go version)
+      if: ${{ env.INITIAL_WORKFLOW_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
+      working-directory: ${{ env.TARGET_REPO_DIR }}
+      shell: bash
+      run: |
+        gofmt -s -w .
+        if ! git diff --quiet; then
+          git add .
+          git commit -m "run gofmt -s"
+        fi

.github/actions/copy-workflow-versioning/action.yml

@@ -0,0 +1,18 @@
+name: copy workflow versioning
+description: Copy workflow steps specific to versioning
+
+runs:
+  using: "composite"
+  steps:
+    - name: add version.json file (in order to deploy versioning workflows)
+      if: hashFiles(format('{0}/version.json', env.TARGET_REPO_DIR)) == ''
+      working-directory: ${{ env.TARGET_REPO_DIR }}
+      shell: bash
+      run: |
+        git fetch origin --unshallow # we need the entire commit history
+        version=$(git describe --tags --abbrev=0 || true) # highest released version on current branch
+        if [[ -n "$version" ]]; then # only deply version.json if there's at least one release
+          printf '{"version": "%s"}' "$version" | jq . > version.json
+          git add version.json
+          git commit -m "add version.json file"
+        fi

.github/workflows/check-3rd-party.yml

@@ -1,8 +1,7 @@
 on: 
   pull_request:
     paths-ignore:
-      - 'config.json'
-      - 'config-testing.json'
+      - 'configs/*.json'
 
 name: Check 3rd Party
 

.github/workflows/check-config.sh

@@ -6,11 +6,11 @@ file=$1
 entries=$(mktemp)
 entries_sorted=$(mktemp)
 
-jq -r ".[].target" $file > $entries
+jq -r ".repositories[].target" $file > $entries
 sort -u $entries > $entries_sorted
 status=0
 if ! output=$(diff -y $entries $entries_sorted); then
-  echo "Targets in config.json not sorted alphabetically:"
+  echo "Targets in $file not sorted alphabetically:"
   echo "$output"
   status=1
 fi

.github/workflows/check-config.yml

@@ -1,7 +1,7 @@
 on: 
   pull_request:
     paths:
-      - 'config.json'
+      - 'configs/*.json'
 
 name: Check Config
 
@@ -12,5 +12,8 @@ jobs:
       targets: ${{ steps.set-matrix.outputs.targets }}
     steps:
       - uses: actions/checkout@v2
-      - name: check if config.json is sorted alphabetically
-        run: .github/workflows/check-config.sh config.json
+      - name: check if configs/*.json is sorted alphabetically
+        run: |
+          for config in configs/*.json; do
+            .github/workflows/check-config.sh $config
+          done

.github/workflows/copy-workflow.yml

@@ -8,9 +8,6 @@ on:
       head_commit_url:
         description: "github.event.head_commit.url of the dispatcher"
         required: true
-      files:
-        description: "List of files to deploy"
-        required: true
       targets:
         description: "List of repositories to deploy to"
         required: true
@@ -28,7 +25,7 @@ jobs:
       TEMPLATE_REPO_DIR: "template-repo"
       TEMPLATE_DIR: "templates"
       NEEDS_UPDATE: 0
-      INITIAL_TEST_DEPLOYMENT: 0
+      INITIAL_WORKFLOW_DEPLOYMENT: 0
       GO_VERSION_BUMP: 0
       FILES: ""
       GITHUB_USER: "web3-bot"
@@ -48,11 +45,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: ${{ env.TEMPLATE_REPO_DIR }}
-    - uses: actions/setup-go@v2
-      with:
-        # This should be the same Go version we use in the go-check workflow.
-        # go mod tidy, go vet, staticcheck and gofmt might behave differently depending on the version.
-        go-version: "1.17.x"
     - name: determine GitHub default branch
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
@@ -63,105 +55,56 @@ jobs:
       run: |
         git config user.name ${{ env.GITHUB_USER }}
         git config user.email ${{ env.GITHUB_EMAIL }}
-    - name: is initial test workflow deployment
+    - name: determine files to add
+      # By setting the environment variable, it's possible to programmatically add / modify this list.
+      # See https://github.com/protocol/.github/blob/38135c75e47839623bf9b2748275d8c6167a8fa8/.github/workflows/copy-workflow.yml#L163-L168 for an example, how we used to make use of this.
       run: |
-        if [[ ! -f $TARGET_REPO_DIR/.github/workflows/go-test.yml ]]; then
-          echo "INITIAL_TEST_DEPLOYMENT=1" >> $GITHUB_ENV
-        fi
-    - name: remove Travis (on initial deployment)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+        files=${{ toJson(toJson(matrix.cfg.files)) }}
+        files=$(echo -e "$files" | jq -c '.')
+        echo "FILES=$files" >> $GITHUB_ENV
+    - name: is initial workflow deployment
+      run: |
+        INITIAL_WORKFLOW_DEPLOYMENT=1
+        for f in $(jq -r '.[]' <<< $FILES); do
+          if [[ -f $TARGET_REPO_DIR/$f ]]; then
+            INITIAL_WORKFLOW_DEPLOYMENT=0
+            break
+          fi
+        done
+        echo "INITIAL_WORKFLOW_DEPLOYMENT=$INITIAL_WORKFLOW_DEPLOYMENT" >> $GITHUB_ENV
+    - name: remove Travis (on initial workflow deployment)
+      if: ${{ env.INITIAL_WORKFLOW_DEPLOYMENT == 1 }}
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
         if [[ -f .travis.yml ]]; then
           git rm .travis.yml
           git commit -m "disable Travis"
         fi
-    - name: remove CircleCI (on initial deployment)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+    - name: remove CircleCI (on initial workflow deployment)
+      if: ${{ env.INITIAL_WORKFLOW_DEPLOYMENT == 1 }}
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
         if [[ -d .circleci ]]; then
           git rm -r .circleci
           git commit -m "disable CircleCI"
         fi
-    - name: remove gx (on initial deployment)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }}
+    - name: remove gx (on initial workflow deployment)
+      if: ${{ env.INITIAL_WORKFLOW_DEPLOYMENT == 1 }}
       working-directory: ${{ env.TARGET_REPO_DIR }}
       run: |
         if [[ -d .gx ]]; then
           git rm -r .gx
           git commit -m "remove .gx"
         fi
-    - name: add version.json file (in order to deploy versioning workflows)
-      if: hashFiles(format('{0}/version.json', env.TARGET_REPO_DIR)) == ''
-      working-directory: ${{ env.TARGET_REPO_DIR }}
-      run: |
-        git fetch origin --unshallow # we need the entire commit history
-        version=$(git describe --tags --abbrev=0 || true) # highest released version on current branch
-        if [[ -n "$version" ]]; then # only deply version.json if there's at least one release
-          printf '{"version": "%s"}' "$version" | jq . > version.json
-          git add version.json
-          git commit -m "add version.json file"
-        fi
-    - name: bump go.mod go version if needed
-      uses: protocol/multiple-go-modules@v1.2
-      with:
-        working-directory: ${{ env.TARGET_REPO_DIR }}
-        run: |
-          # We want our modules to support two Go versions at a time.
-          # As of August 2021, Go 1.17 is the latest stable.
-          # go.mod's Go version declares the language version being used.
-          # As such, it has to be the minimum of all Go versions supported.
-          # Bump this every six months, as new Go versions come out.
-          TARGET_VERSION=1.16
-
-          # Note that the "<" comparison doesn't understand semver,
-          # but it should be good enough for the foreseeable future.
-          CURRENT_VERSION=$(go list -m -f {{.GoVersion}})
-
-          if [[ $CURRENT_VERSION < $TARGET_VERSION ]]; then
-            echo "GO_VERSION_BUMP=1" >> $GITHUB_ENV
-
-            # Update the version in go.mod. This alone ensures there's a diff.
-            go mod edit -go $TARGET_VERSION
-
-            # In the future, "go fix" may make changes to Go code,
-            # such as to adapt to language changes or API deprecations.
-            # This is largely a no-op as of Go 1.17, and that's fine.
-            go fix ./...
-            git add .
-
-            # We don't tidy, because the next step does that.
-            # Separate commits also help with reviews.
-            git commit -m "bump go.mod to Go $TARGET_VERSION and run go fix"
-          fi
-    - name: go mod tidy (on initial deployment and on new Go version)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
-      uses: protocol/multiple-go-modules@v1.2
-      with:
-        working-directory: ${{ env.TARGET_REPO_DIR }}
-        run: |
-          go mod tidy
-          if ! git diff --quiet; then
-            git add .
-            git commit -m "run go mod tidy"
-          fi
-    - name: gofmt -s (on initial deployment and on new Go version)
-      if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}}
-      working-directory: ${{ env.TARGET_REPO_DIR }}
-      run: |
-        gofmt -s -w .
-        if ! git diff --quiet; then
-          git add .
-          git commit -m "run gofmt -s"
-        fi
-    - name: determine files to add
-      # By setting the environment variable, it's possible to programmatically add / modify this list.
-      # See https://github.com/protocol/.github/blob/38135c75e47839623bf9b2748275d8c6167a8fa8/.github/workflows/copy-workflow.yml#L163-L168 for an example, how we used to make use of this.
-      run: echo "FILES=${{ toJson(github.event.inputs.files) }}" >> $GITHUB_ENV
+    - name: Run steps specific to go
+      if: matrix.cfg.deploy_go
+      uses: ./template-repo/.github/actions/copy-workflow-go
+    - name: Run steps specific to versioning
+      if: matrix.cfg.deploy_versioning
+      uses: ./template-repo/.github/actions/copy-workflow-versioning
     - name: Add files
       run: |
-        for f in $(jq -r ".[]" <<< ${{ toJson(env.FILES) }}); do
+        for f in $(jq -r '.[]' <<< $FILES); do
           echo -e "\nProcessing $f."
           # add DO NOT EDIT header
           tmp=$(mktemp)

.github/workflows/dispatch.yml

@@ -12,43 +12,53 @@ env:
   # With ~180 repos, this means we'll run around 9 instances of the copy workflow in parallel.
   # This is (hopefully) sufficient to prevent us from triggering GitHub Action's abuse detection mechanism.
   MAX_REPOS_PER_WORKFLOW: 20
-  FILES: '[ ".github/workflows/automerge.yml", ".github/workflows/go-test.yml", ".github/workflows/go-check.yml", ".github/workflows/releaser.yml", ".github/workflows/release-check.yml", ".github/workflows/tagpush.yml" ]' # a JSON array of the files to distribute
 
 jobs:
   matrix:
     name: Trigger copy workflows
     runs-on: ubuntu-latest
     outputs:
-      targets: ${{ steps.set-matrix.outputs.targets }}
+      batches: ${{ steps.set-matrix.outputs.batches }}
     steps:
       - uses: actions/checkout@v2
       - id: set-matrix
         run: |
-          CONFIG=config.json
-          if [[ $GITHUB_REF == refs/heads/testing ]]; then
-            CONFIG=config-testing.json
-          fi
-          TARGETS=$(jq '. | _nwise(${{ env.MAX_REPOS_PER_WORKFLOW }})' $CONFIG | jq -sc '. | to_entries')
-          echo "::set-output name=targets::$TARGETS"
+          TARGETS=()
+          for config in configs/*.json; do
+            if [[ $GITHUB_REF == refs/heads/testing && $config != configs/testing.json ]]; then
+              continue
+            fi
+            if [[ $GITHUB_REF != refs/heads/testing && $config == configs/testing.json ]]; then
+              continue
+            fi
+            DEFAULTS=$(jq -c '.defaults' $config)
+            TARGETS+=($(jq -c ".repositories[] | $DEFAULTS + ." $config))
+          done
+          BATCHES=$(jq -sc '[. | _nwise(${{ env.MAX_REPOS_PER_WORKFLOW }})] | to_entries' <<< $TARGETS)
+          echo "::set-output name=batches::$BATCHES"
   dispatch:
     needs: [ matrix ]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         # We end up with multiple "dispatch" jobs,
-        # one per TARGETS "key" chunk above with a "value" array.
+        # one per BATCHES "key" chunk above with a "value" array.
         # For each "dispatch" job, matrix.cfg.value is an array, like:
         #
-        #   [{"target": "repo1", "target": "repo2"}]
+        #   [
+        #     {"target": "repo1", "files": [".github/workflows/go-check.yml"]}, 
+        #     {"target": "repo2", "files": [".github/workflows/go-check.yml", ".github/workflows/go-test.yml"]}
+        #   ]
         #
         # The triggered copy-workflow jobs use that final array as their matrix.
         # As such, we'll end up with one copy-workflow parallel job per target.
-        cfg: ${{ fromJson(needs.matrix.outputs.targets) }}
+        cfg: ${{ fromJson(needs.matrix.outputs.batches) }}
     name: Start copy workflow (batch ${{ matrix.cfg.key }})
     steps:
       - uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1.1.0
         with:
           workflow: "Deploy" # "name" attribute of copy-workflow.yml
           token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }}
-          inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "files": ${{ toJson(env.FILES) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }'
+          # double toJson on matrix.cfg.value is here on purpose
+          inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }'