Add changelog generation

.github/workflows/draft_release.yml

@@ -0,0 +1,207 @@
+name: Publish draft release
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref1:
+        description: The 'from' tag to use for the diff
+        default: statemine-v5.0.0
+        required: true
+      ref2:
+        description: The 'to' tag to use for the diff
+        default: HEAD
+        required: true
+      pre_release:
+        description: For pre-releases
+        default: "true"
+        required: true
+
+jobs:
+  get-rust-versions:
+    runs-on: ubuntu-latest
+    container:
+      image: paritytech/ci-linux:production
+    outputs:
+      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
+      rustc-nightly: ${{ steps.get-rust-versions.outputs.nightly }}
+    steps:
+      - id: get-rust-versions
+        run: |
+          echo "::set-output name=stable::$(rustc +stable --version)"
+          echo "::set-output name=nightly::$(rustc +nightly --version)"
+
+  build-runtimes:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runtime: ["shell", "statemine", "statemint", "westmint", "rococo"]
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Cache target dir
+        uses: actions/cache@v2
+        with:
+          path: "${{ github.workspace }}/runtime/${{ matrix.runtime }}/target"
+          key: srtool-target-${{ matrix.runtime }}-${{ github.sha }}
+          restore-keys: |
+            srtool-target-${{ matrix.runtime }}-
+            srtool-target-
+
+      - name: Build ${{ matrix.runtime }} runtime
+        id: srtool_build
+        uses: chevdor/srtool-actions@v0.3.0
+        with:
+          image: paritytech/srtool
+          chain: ${{ matrix.runtime }}
+          runtime_dir: polkadot-parachains/${{ matrix.runtime }}
+
+      - name: Store srtool digest to disk
+        run: |
+          echo '${{ steps.srtool_build.outputs.json }}' | \
+            jq > ${{ matrix.runtime }}_srtool_output.json
+
+      - name: Upload ${{ matrix.runtime }} srtool json
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.runtime }}-srtool-json
+          path: ${{ matrix.runtime }}_srtool_output.json
+
+      - name: Upload ${{ matrix.runtime }} runtime
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.runtime }}-runtime
+          path: |
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+
+  publish-draft-release:
+    runs-on: ubuntu-latest
+    needs: ["get-rust-versions", "build-runtimes"]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          path: cumulus
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0.0
+
+      - name: Download srtool json output
+        uses: actions/download-artifact@v2
+
+      - name: Prepare tooling
+        run: |
+          cd cumulus/scripts/changelog
+          gem install bundler changelogerator
+          bundle install
+          changelogerator --help
+
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.1/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Generate release notes
+        env:
+          RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
+          RUSTC_NIGHTLY: ${{ needs.get-rust-versions.outputs.rustc-nightly }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NO_CACHE: 1
+          DEBUG: 1
+          SHELL_DIGEST: ${{ github.workspace}}/shell-srtool-json/shell_srtool_output.json
+          WESTMINT_DIGEST: ${{ github.workspace}}/westmint-srtool-json/westmint_srtool_output.json
+          STATEMINE_DIGEST: ${{ github.workspace}}/statemine-srtool-json/statemine_srtool_output.json
+          STATEMINT_DIGEST: ${{ github.workspace}}/statemint-srtool-json/statemint_srtool_output.json
+          ROCOCO_DIGEST: ${{ github.workspace}}/rococo-srtool-json/rococo_srtool_output.json
+          REF1: ${{ github.event.inputs.ref1 }}
+          REF2: ${{ github.event.inputs.ref2 }}
+          PRE_RELEASE: ${{ github.event.inputs.pre_release }}
+          HIDE_SRTOOL_ROCOCO: false
+          HIDE_SRTOOL_SHELL: false
+        run: |
+          find ${{env.GITHUB_WORKSPACE}} -type f -name "*_srtool_output.json"
+          ls -al $SHELL_DIGEST
+          ls -al $WESTMINT_DIGEST
+          ls -al $STATEMINE_DIGEST
+          ls -al $STATEMINT_DIGEST
+          ls -al $ROCOCO_DIGEST
+
+          echo "The diff will be computed from $REF1 to $REF2"
+          cd cumulus/scripts/changelog
+          ./bin/changelog $REF1 $REF2 release-notes.md
+          ls -al release-notes.md
+
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: parachains-${{ github.ref }}
+          release_name: Parachains ${{ github.ref }}
+          body_path: ./cumulus/scripts/changelog/release-notes.md
+          draft: true
+
+  publish-runtimes:
+    runs-on: ubuntu-latest
+    needs: ["publish-draft-release"]
+    env:
+      RUNTIME_DIR: polkadot-parachains
+    strategy:
+      matrix:
+        runtime: ["shell", "statemine", "statemint", "westmint", "rococo"]
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0.0
+
+      - name: Get runtime version for ${{ matrix.runtime }}
+        id: get-runtime-ver
+        run: |
+          echo "require './scripts/github/runtime-version.rb'" > script.rb
+          echo "puts get_runtime(runtime: \"${{ matrix.runtime }}\", runtime_dir: \"$RUNTIME_DIR\")" >> script.rb
+
+          echo "Current folder: $PWD"
+          ls "$RUNTIME_DIR/${{ matrix.runtime }}"
+          runtime_ver=$(ruby script.rb)
+          echo "Found version: >$runtime_ver<"
+          echo "::set-output name=runtime_ver::$runtime_ver"
+
+      - name: Upload compressed ${{ matrix.runtime }} wasm
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.publish-draft-release.outputs.asset_upload_url }}
+          asset_path: "${{ matrix.runtime }}-runtime/${{ matrix.runtime }}_runtime.compact.compressed.wasm"
+          asset_name: ${{ matrix.runtime }}_runtime-v${{ steps.get-runtime-ver.outputs.runtime_ver }}.compact.compressed.wasm
+          asset_content_type: application/wasm
+
+  post_to_matrix:
+    runs-on: ubuntu-latest
+    needs: publish-draft-release
+    steps:
+      - name: Internal polkadot channel
+        uses: s3krit/matrix-message-action@v0.0.3
+        with:
+          room_id: ${{ secrets.INTERNAL_POLKADOT_MATRIX_ROOM_ID }}
+          access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}
+          message: |
+            **New draft for ${{ github.repository }}**: ${{ github.ref }}<br/>
+
+            Draft release created: [draft](${{ needs.publish-draft-release.outputs.release_url }})
+
+            NOTE: The link above will no longer be valid if the draft is edited. You can then use the following link:
+            [${{ github.server_url }}/${{ github.repository }}/releases](${{ github.server_url }}/${{ github.repository }}/releases)
+          server: "matrix.parity.io"

scripts/changelog/.gitignore

@@ -1 +1,4 @@
 changelog.md
+*.json
+release*.md
+.env

scripts/changelog/Gemfile

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gem 'octokit', '~> 4'
+
+gem 'git_diff_parser', '~> 3'
+
+gem 'toml', '~> 0.3.0'
+
+gem 'rake', group: :dev
+
+gem 'optparse', '~> 0.1.1'
+
+gem 'logger', '~> 1.4'
+
+gem 'test-unit', group: :dev
+
+gem 'rubocop', group: :dev, require: false

scripts/changelog/Gemfile.lock

@@ -0,0 +1,79 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    git_diff_parser (3.2.0)
+    logger (1.4.4)
+    multipart-post (2.1.1)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    optparse (0.1.1)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    parslet (2.0.0)
+    power_assert (2.0.1)
+    public_suffix (4.0.6)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.23.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    test-unit (3.5.1)
+      power_assert
+    toml (0.3.0)
+      parslet (>= 1.8.0, < 3.0.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  git_diff_parser (~> 3)
+  logger (~> 1.4)
+  octokit (~> 4)
+  optparse (~> 0.1.1)
+  rake
+  rubocop
+  test-unit
+  toml (~> 0.3.0)
+
+BUNDLED WITH
+   2.2.22

scripts/changelog/README.md

@@ -1,13 +1,71 @@
 # Changelog
 
-Currently, the changelog is built locally.
-Run:
+Currently, the changelog is built locally. It will be moved to CI once labels stabilize.
+
+For now, a bit of preparation is required before you can run the script:
+- fetch the srtool digests
+- store them under the `digests` folder as `<chain>-srtool-digest.json`
+- ensure the `.env` file is up to date with correct information
+
+The content of the release notes is generated from the template files under the `scripts/changelog/templates` folder. For readability and maintenance, the template is split into several small snippets.
 
+Run:
 ```
-./changelog.sh <ref_since>
+./bin/changelog <ref_since> [<ref_until>=HEAD]
 ```
 
 For instance:
 ```
-./changelog.sh statemine_v4
+./bin/changelog statemine-v5.0.0
+```
+
+A file called `release-notes.md` will be generated and can be used for the release.
+
+## ENV
+
+You may use the following ENV for testing:
+
+```
+RUSTC_STABLE="rustc 1.56.1 (59eed8a2a 2021-11-01)"
+RUSTC_NIGHTLY="rustc 1.57.0-nightly (51e514c0f 2021-09-12)"
+PRE_RELEASE=true
+HIDE_SRTOOL_ROCOCO=true
+HIDE_SRTOOL_SHELL=true
+REF1=statemine-v5.0.0
+REF2=HEAD
+DEBUG=1
+NO_CACHE=1
 ```
+## Considered labels
+
+The following list will likely evolve over time and it will be hard to keep it in sync.
+In any case, if you want to find all the labels that are used, search for `meta` in the templates.
+Currently, the considered labels are:
+
+- Priority: C<N> labels
+- Audit: D<N> labels
+- E4 => new host function
+- B0 => silent, not showing up
+- B1-releasenotes (misc unless other labels)
+- B5-client (client changes)
+- B7-runtimenoteworthy (runtime changes)
+- T6-XCM
+
+Note that labels with the same letter are mutually exclusive.
+A PR should not have both `B0` and `B5`, or both `C1` and `C9`. In case of conflicts, the template will
+decide which label will be considered.
+
+## Dev and debuggin
+
+### Hot Reload
+
+The following command allows **Hot Reload**:
+```
+fswatch templates -e ".*\.md$" | xargs -n1 -I{} ./bin/changelog statemine-v5.0.0
+```
+### Caching
+
+By default, if the changelog data from Github is already present, the calls to the Github API will be skipped
+and the local version of the data will be used. This is much faster.
+If you know that some labels have changed in Github, you probably want to refresh the data.
+You can then either delete manually the `cumulus.json` file or `export NO_CACHE=1` to force refreshing the data.