Rework CI to use Chopsticks servers to reduce RPC load

.github/workflows/ci.yml

@@ -25,20 +25,144 @@
   define-matrix:
     runs-on: ubuntu-latest
     outputs:
-      tests: ${{ steps.tests.outputs.tests }}
+      pah-tests: ${{ steps.tests.outputs.pah-tests }}
+      kah-tests: ${{ steps.tests.outputs.kah-tests }}
+      other-tests: ${{ steps.tests.outputs.other-tests }}
     steps:
       - uses: actions/checkout@v4
       - name: Define Tests
         id: tests
         run: |
-          echo tests=$(cd packages && ls */src/*.test.ts | jq -R -s -c 'split("\n")[:-1]') >> "$GITHUB_OUTPUT"
-  tests:
+          cd packages
+          all_tests=$(ls */src/*.test.ts)
+
+          # Filter PAH tests (assetHubPolkadot)
+          pah_tests=$(echo "$all_tests" | grep -i "assetHubPolkadot" || true)
+          echo "pah-tests=$(echo "$pah_tests" | jq -R -s -c 'split("\n") | map(select(length > 0))')" >> "$GITHUB_OUTPUT"
+
+          # Filter KAH tests (assetHubKusama)
+          kah_tests=$(echo "$all_tests" | grep -i "assetHubKusama" || true)
+          echo "kah-tests=$(echo "$kah_tests" | jq -R -s -c 'split("\n") | map(select(length > 0))')" >> "$GITHUB_OUTPUT"
+
+          # All other tests
+          other_tests=$(echo "$all_tests" | grep -v -i "assetHub" || true)
+          echo "other-tests=$(echo "$other_tests" | jq -R -s -c 'split("\n") | map(select(length > 0))')" >> "$GITHUB_OUTPUT"
+
+  pah-tests:
+    needs: define-matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: setup node env
+      uses: actions/setup-node@v4
+      with:
+        node-version: 22.x
+        cache: 'yarn'
+    - run: yarn --immutable
+    - name: Start Chopsticks for PAH
+      run: |
+        source KNOWN_GOOD_BLOCK_NUMBERS_POLKADOT.env
+        npx @acala-network/chopsticks@latest \
+          --endpoint=wss://sys.ibp.network/asset-hub-polkadot \
+          --block=$ASSETHUBPOLKADOT_BLOCK_NUMBER \
+          --port=8001 > /tmp/chopsticks-pah.log 2>&1 &
+        echo $! > /tmp/chopsticks-pah.pid
+
+        # Wait for server to be ready
+        timeout=120
+        elapsed=0
+        while [ $elapsed -lt $timeout ]; do
+          if nc -z localhost 8001 2>/dev/null; then
+            echo "Chopsticks PAH server ready"
+            break
+          fi
+          sleep 2
+          elapsed=$((elapsed + 2))
+        done
+
+        if [ $elapsed -ge $timeout ]; then
+          echo "Timeout waiting for chopsticks"
+          cat /tmp/chopsticks-pah.log
+          exit 1
+        fi
+    - name: Run PAH Tests
+      env:
+        ASSETHUBPOLKADOT_ENDPOINT: ws://localhost:8001
+      run: |
+        tests='${{ needs.define-matrix.outputs.pah-tests }}'
+        echo "$tests" | jq -r '.[]' | while read test; do
+          echo "Running test: $test"
+          yarn test "packages/$test" || exit 1
+        done
+    - name: Cleanup
+      if: always()
+      run: |
+        if [ -f /tmp/chopsticks-pah.pid ]; then
+          kill $(cat /tmp/chopsticks-pah.pid) || true
+        fi
+
+  kah-tests:
+    needs: define-matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: setup node env
+      uses: actions/setup-node@v4
+      with:
+        node-version: 22.x
+        cache: 'yarn'
+    - run: yarn --immutable
+    - name: Start Chopsticks for KAH
+      run: |
+        source KNOWN_GOOD_BLOCK_NUMBERS_KUSAMA.env
+        npx @acala-network/chopsticks@latest \
+          --endpoint=wss://sys.ibp.network/asset-hub-kusama \
+          --block=$ASSETHUBKUSAMA_BLOCK_NUMBER \
+          --port=8002 > /tmp/chopsticks-kah.log 2>&1 &
+        echo $! > /tmp/chopsticks-kah.pid
+
+        # Wait for server to be ready
+        timeout=120
+        elapsed=0
+        while [ $elapsed -lt $timeout ]; do
+          if nc -z localhost 8002 2>/dev/null; then
+            echo "Chopsticks KAH server ready"
+            break
+          fi
+          sleep 2
+          elapsed=$((elapsed + 2))
+        done
+
+        if [ $elapsed -ge $timeout ]; then
+          echo "Timeout waiting for chopsticks"
+          cat /tmp/chopsticks-kah.log
+          exit 1
+        fi
+    - name: Run KAH Tests
+      env:
+        ASSETHUBKUSAMA_ENDPOINT: ws://localhost:8002
+      run: |
+        tests='${{ needs.define-matrix.outputs.kah-tests }}'
+        echo "$tests" | jq -r '.[]' | while read test; do
+          echo "Running test: $test"
+          yarn test "packages/$test" || exit 1
+        done
+    - name: Cleanup
+      if: always()
+      run: |
+        if [ -f /tmp/chopsticks-kah.pid ]; then
+          kill $(cat /tmp/chopsticks-kah.pid) || true
+        fi
+
+  other-tests:
     needs: define-matrix
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        tests: ${{ fromJSON(needs.define-matrix.outputs.tests) }}
+        tests: ${{ fromJSON(needs.define-matrix.outputs.other-tests) }}
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -49,9 +173,10 @@
         cache: 'yarn'
     - run: yarn --immutable
     - run: yarn test packages/${{ matrix.tests }}
+
   all-passed:
     runs-on: ubuntu-latest
-    needs: tests
+    needs: [pah-tests, kah-tests, other-tests]
     if: always()
     steps:
       - name: All tests ok

.github/workflows/update-snapshot.yml

@@ -27,7 +27,56 @@ jobs:
         cache: 'yarn'
     - run: yarn --immutable
     - run: yarn update-known-good
-    - run: yarn test:${{ matrix.network }} -u
+    - name: Start Chopsticks
+      run: |
+        if [ "${{ matrix.network }}" = "polkadot" ]; then
+          source KNOWN_GOOD_BLOCK_NUMBERS_POLKADOT.env
+          npx @acala-network/chopsticks@latest \
+            --endpoint=wss://sys.ibp.network/asset-hub-polkadot \
+            --block=$ASSETHUBPOLKADOT_BLOCK_NUMBER \
+            --port=8001 > /tmp/chopsticks.log 2>&1 &
+          echo $! > /tmp/chopsticks.pid
+          PORT=8001
+          ENDPOINT_VAR="ASSETHUBPOLKADOT_ENDPOINT"
+        else
+          source KNOWN_GOOD_BLOCK_NUMBERS_KUSAMA.env
+          npx @acala-network/chopsticks@latest \
+            --endpoint=wss://sys.ibp.network/asset-hub-kusama \
+            --block=$ASSETHUBKUSAMA_BLOCK_NUMBER \
+            --port=8002 > /tmp/chopsticks.log 2>&1 &
+          echo $! > /tmp/chopsticks.pid
+          PORT=8002
+          ENDPOINT_VAR="ASSETHUBKUSAMA_ENDPOINT"
+        fi
+
+        # Wait for server to be ready
+        timeout=120
+        elapsed=0
+        while [ $elapsed -lt $timeout ]; do
+          if nc -z localhost $PORT 2>/dev/null; then
+            echo "Chopsticks server ready on port $PORT"
+            break
+          fi
+          sleep 2
+          elapsed=$((elapsed + 2))
+        done
+
+        if [ $elapsed -ge $timeout ]; then
+          echo "Timeout waiting for chopsticks"
+          cat /tmp/chopsticks.log
+          exit 1
+        fi
+
+        # Export endpoint for tests
+        echo "${ENDPOINT_VAR}=ws://localhost:${PORT}" >> $GITHUB_ENV
+    - name: Run Tests
+      run: yarn test:${{ matrix.network }} -u
+    - name: Cleanup
+      if: always()
+      run: |
+        if [ -f /tmp/chopsticks.pid ]; then
+          kill $(cat /tmp/chopsticks.pid) || true
+        fi
     - name: Commit and Create PR
       uses: actions/github-script@v6
       with:

KNOWN_GOOD_BLOCK_NUMBERS_KUSAMA.env

@@ -1,10 +1,10 @@
-ASSETHUBKUSAMA_BLOCK_NUMBER=12792874
-BASILISK_BLOCK_NUMBER=12909953
-BRIDGEHUBKUSAMA_BLOCK_NUMBER=7419039
-CORETIMEKUSAMA_BLOCK_NUMBER=4353209
-ENCOINTERKUSAMA_BLOCK_NUMBER=12255934
-KARURA_BLOCK_NUMBER=10794731
-KUSAMA_BLOCK_NUMBER=31925684
-MOONRIVER_BLOCK_NUMBER=14821033
-PEOPLEKUSAMA_BLOCK_NUMBER=7485670
-SHIDEN_BLOCK_NUMBER=13432313
+ASSETHUBKUSAMA_BLOCK_NUMBER=12881073
+BASILISK_BLOCK_NUMBER=13004209
+BRIDGEHUBKUSAMA_BLOCK_NUMBER=7470366
+CORETIMEKUSAMA_BLOCK_NUMBER=4404093
+ENCOINTERKUSAMA_BLOCK_NUMBER=12350236
+KARURA_BLOCK_NUMBER=10844230
+KUSAMA_BLOCK_NUMBER=32028925
+MOONRIVER_BLOCK_NUMBER=14918200
+PEOPLEKUSAMA_BLOCK_NUMBER=7587604
+SHIDEN_BLOCK_NUMBER=13526493

KNOWN_GOOD_BLOCK_NUMBERS_POLKADOT.env

@@ -1,10 +1,10 @@
 ACALA_BLOCK_NUMBER=10230485
-ASSETHUBPOLKADOT_BLOCK_NUMBER=11322985
-ASTAR_BLOCK_NUMBER=11983100
-BRIDGEHUBPOLKADOT_BLOCK_NUMBER=6860782
-COLLECTIVESPOLKADOT_BLOCK_NUMBER=8059510
-CORETIMEPOLKADOT_BLOCK_NUMBER=3452814
-HYDRATION_BLOCK_NUMBER=11036597
-MOONBEAM_BLOCK_NUMBER=14171137
-PEOPLEPOLKADOT_BLOCK_NUMBER=3792580
-POLKADOT_BLOCK_NUMBER=29609476
+ASSETHUBPOLKADOT_BLOCK_NUMBER=11454539
+ASTAR_BLOCK_NUMBER=12075809
+BRIDGEHUBPOLKADOT_BLOCK_NUMBER=6912080
+COLLECTIVESPOLKADOT_BLOCK_NUMBER=8111642
+CORETIMEPOLKADOT_BLOCK_NUMBER=3503876
+HYDRATION_BLOCK_NUMBER=11139081
+MOONBEAM_BLOCK_NUMBER=14264300
+PEOPLEPOLKADOT_BLOCK_NUMBER=3844323
+POLKADOT_BLOCK_NUMBER=29714091