Set of fixes for test runtimes and various minor issues (#73)

Fix WASM filename for test runtimes and runtimes with dashes in name

fix #71
fix #72

* Scan and build all SDK runtimes
* Delete the cumulus specific workflow since it is now part of the SDK
* Checkout both srtool and the sdk repo
* Fix path
* Fix checkout dir
* Fix working directory
* Allow customizing the image and fix paths
* Test with srtool-actions with more debugging info
* Set global working_directory for run commands
- bash fmt
- add support for ruuntime exclusions
- back to the latest subwasm version: 0.20.0
* Remove default EXCLUDED_RUNTIMES
* Fix subwasm results path
* Gate subwasm differ based on the existence of the reference chain
* Add optional caching support
* Don't fail when reference chain is not found
* Optimize caching
* Fix reference chain check
* Fix caching input
* Store artifacts to disk
* Get data from cache
* Remove extra quotes
* Fix dashes
* Fix runtimes-list caching
* Make sure to use the cached runtimes list when needed
* Fix working_dir for wasm files
* Add caching for the fellowship builds
* Fix Kusama package name
* Version bump
* Rename Fellow workflow
* Add nscan script to scan for runtimes
* Cleanup and doc
* Fix Kusama name

.github/workflows/manual-fellow-runtimes.yml

@@ -1,11 +1,14 @@
-name: Runtimes Fellowship
+name: Manual Build - Fellowship
 
 env:
   SUBWASM_VERSION: 0.20.0
 
 on:
   workflow_dispatch:
     inputs:
+      image:
+        description: The SRTOOL image to use use
+        default: paritytech/srtool
       srtool_tag:
         description: The SRTOOL tag to use
         default: 1.70.0
@@ -14,6 +17,9 @@ on:
         description: The ref to be used for the repo
         default: main
         required: false
+      cache:
+        description: By default, caching will be used but you can turn it off here if you provide 'false'
+        default: true
   schedule:
     - cron: "00 04 * * 1"
 
@@ -26,8 +32,8 @@ jobs:
         include:
           # Relay
           - runtime_dir: relay/kusama
-            runtime: kusama-runtime
-            chain: kusama
+            runtime: staging-kusama-runtime
+            chain: staging-kusama
           - runtime_dir: relay/polkadot
             runtime: polkadot-runtime
             chain: polkadot
@@ -55,25 +61,65 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
+      - name: Cache ${{ matrix.chain }} runtime for ${{ github.sha }}
+        id: cache_runtime
+        if: github.event.inputs.cache == 'true'
+        uses: actions/cache@v3
+        with:
+          key: ${{ matrix.chain }}-${{ github.sha }}
+          path: |
+            sdk/${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime
+
       - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
         with:
           repository: polkadot-fellows/runtimes
           ref: ${{ github.event.inputs.ref }}
           fetch-depth: 0
 
       - name: Srtool build
         id: srtool_build
+        if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
         uses: chevdor/[email protected]
         with:
           chain: ${{ matrix.chain }}
-          tag: ${{ github.event.inputs.srtool_tag }}
           runtime_dir:  ${{ matrix.runtime_dir }}
+          image: ${{ github.event.inputs.image }}
+          tag: ${{ github.event.inputs.srtool_tag }}
+
+      # This is done to allow caching
+      - name: Store build artifacts to disk
+        id: cache_digest
+        if: ${{ steps.cache_runtime.outputs.cache-hit != 'true' }}
+        run: |
+          cached_output=${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime/
+          digest_file=${cached_output}/${{ matrix.chain }}-srtool-digest.json
+          echo '${{ steps.srtool_build.outputs.json }}' | jq > ${digest_file}
+          cat ${digest_file}
+          echo "digest_file=$digest_file" >> "$GITHUB_OUTPUT"
+
+      - name: Retrieve WASM paths
+        id: get_values
+        run: |
+          runtime_package=${{ matrix.chain }}-runtime
+          runtime_filename=${runtime_package//-/_}
+
+          wasm="${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${runtime_package}/${runtime_filename}.compact.wasm"
+
+          wasm_compressed="${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${runtime_package}/${runtime_filename}.compact.compressed.wasm"
+
+          # we need wasm and wasm_compressed
+          echo "wasm=$wasm" >> "$GITHUB_OUTPUT"
+          echo "wasm_compressed=$wasm_compressed" >> "$GITHUB_OUTPUT"
 
       - name: Summary
         run: |
-          echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json
-          cat ${{ matrix.chain }}-srtool-digest.json
-          echo "Runtime location: ${{ steps.srtool_build.outputs.wasm }}"
+          cached_output=${{ matrix.runtime_dir }}/target/srtool/release/wbuild/${{ matrix.chain }}-runtime/
+          digest_file=${cached_output}/${{ matrix.chain }}-srtool-digest.json
+          ls -al ${digest_file}
+          cat ${digest_file}
+          echo "WASM location: ${{ steps.get_values.outputs.wasm }}"
+          echo "WASM Compressed location: ${{ steps.get_values.outputs.wasm_compressed }}"
 
       # it takes a while to build the runtime, so let's save the artifact as soon as we have it
       - name: Archive Artifacts for ${{ matrix.chain }}
@@ -91,6 +137,7 @@ jobs:
           wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
           sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
           subwasm --version
+
       - name: Show Runtime information
         run: |
           subwasm info ${{ steps.srtool_build.outputs.wasm }}