diff --git a/.github/workflows/rpc-parity-report.yml b/.github/workflows/rpc-parity-report.yml new file mode 100644 index 000000000000..f38f8bf5b342 --- /dev/null +++ b/.github/workflows/rpc-parity-report.yml @@ -0,0 +1,65 @@ +name: RPC parity conformance report +on: + workflow_dispatch: + inputs: + image: + description: "Forest Docker image to test against" + required: false + default: "ghcr.io/chainsafe/forest:edge-fat" + schedule: + - cron: "0 0 1 * *" # Monthly, on the first day of the month at midnight +jobs: + rpc-parity: + name: RPC parity report + runs-on: buildjet-8vcpu-ubuntu-2204 + steps: + - name: Relocate docker volumes folder + run: | + # move docker volumes folder to under `/mnt` which has 60GB+ free space + sudo ls /var/lib/docker/volumes + sudo mv /var/lib/docker/volumes /mnt/docker-volumes + sudo ln -sf /mnt/docker-volumes /var/lib/docker/volumes + sudo ls /var/lib/docker/volumes + df -h + - uses: actions/checkout@v6 + - name: Run api compare tests on calibnet + shell: bash + run: | + IMAGE=${{ github.event.inputs.image }} + if [ -z "$IMAGE" ]; then + IMAGE="ghcr.io/chainsafe/forest:edge-fat" + fi + echo "FROM $IMAGE" > Dockerfile-RPC + export FOREST_DOCKERFILE_OVERRIDE=Dockerfile-RPC + ./scripts/tests/api_compare/setup.sh + timeout-minutes: 30 + - name: Dump docker logs + if: always() + uses: jwalton/gh-docker-logs@v2 + - name: Dump Docker usage + if: always() + run: | + sudo ls /var/lib/docker/volumes + docker system df + docker system df --verbose + df -h + - uses: jdx/mise-action@v3 + - run: mise add_conformance_report + # This is needed in order to have the commits signed. + - uses: actions/create-github-app-token@v2 + id: generate-token + with: + app-id: ${{ secrets.LESHY_APP_ID }} + private-key: ${{ secrets.LESHY_APP_PRIVATE_KEY }} + - name: Create Pull Request + uses: peter-evans/create-pull-request@v8 + with: + base: main + branch: leshy/scheduled-rpc-parity-report + token: ${{ steps.generate-token.outputs.token }} + commit-message: Add monthly RPC parity conformance report + sign-commits: true + title: "[automated] Add monthly RPC parity conformance report" + body: | + ### Changes + - Adds the monthly RPC parity conformance report to the docs. diff --git a/docs/docs/users/reports/_category_.json b/docs/docs/users/reports/_category_.json new file mode 100644 index 000000000000..89c736b238a2 --- /dev/null +++ b/docs/docs/users/reports/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Reports", + "position": 5 +} diff --git a/docs/docs/users/reports/api_conformance/_category_.json b/docs/docs/users/reports/api_conformance/_category_.json new file mode 100644 index 000000000000..95b1207ac912 --- /dev/null +++ b/docs/docs/users/reports/api_conformance/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "API Conformance Reports", + "position": 1 +} diff --git a/mise-tasks/add_conformance_report.sh b/mise-tasks/add_conformance_report.sh new file mode 100755 index 000000000000..6425ce8377cc --- /dev/null +++ b/mise-tasks/add_conformance_report.sh @@ -0,0 +1,7 @@ +#!/bin/bash +#MISE description="Extracts conformance report from the API compare compose volume and adds it to the docs." + +TMP_REPORT=$(mktemp -d)/report.json +DATE=$(date +%Y-%m-%d) +./scripts/tests/api_compare/extract_conformance_report.sh "$TMP_REPORT" +./scripts/tests/api_compare/convert_report_to_markdown.sh "$TMP_REPORT" ./docs/docs/users/reports/api_conformance/report_"$DATE".md diff --git a/scripts/tests/api_compare/convert_report_to_markdown.sh b/scripts/tests/api_compare/convert_report_to_markdown.sh new file mode 100755 index 000000000000..2e19c9d5ff6d --- /dev/null +++ b/scripts/tests/api_compare/convert_report_to_markdown.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Simple script to generate a markdown report from test.json +# Usage: ./convert_report_to_markdown.sh + +set -euo pipefail + +INPUT_FILE="$1" +OUTPUT_FILE="$2" + +# Check if input file exists +if [[ ! -f "$INPUT_FILE" ]]; then + echo "Error: Input file '$INPUT_FILE' not found." + exit 1 +fi + +# Generate the markdown report +{ + echo "# $(date '+%Y-%m-%d') - API Parity Report" + echo "" + echo "| Method | Lotus-conformance check |" + echo "|--------|-------------------------|" + + jq -r '.methods[] | + if (.name | startswith("Forest.")) then + "| `\(.name)` | N/A (Forest-specific) |" + else + "| `\(.name)` | \(if .status.type == "tested" then "✅" else "❌" end) |" + end' "$INPUT_FILE" + +} > "$OUTPUT_FILE" + +echo "Report generated: $OUTPUT_FILE" diff --git a/scripts/tests/api_compare/extract_conformance_report.sh b/scripts/tests/api_compare/extract_conformance_report.sh new file mode 100755 index 000000000000..7e97e995debb --- /dev/null +++ b/scripts/tests/api_compare/extract_conformance_report.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script extracts the conformance report from the finished compose volume. +# Usage: ./extract_conformance_report.sh + +set -euo pipefail + +OUTPUT_PATH="$1" + +# Volume is in the `node-data` volume +VOLUME_NAME="api_compare_node-data" +TEMP_CONTAINER_NAME="temp-extract-container" + +# Create a temporary container to access the volume +docker run --name "$TEMP_CONTAINER_NAME" -v "$VOLUME_NAME":/data:ro -d alpine sleep infinity +trap 'docker rm -f "$TEMP_CONTAINER_NAME"' EXIT + +REPORT_PATH_IN_VOLUME=$(docker exec "$TEMP_CONTAINER_NAME" sh -c 'find /data/api-compare-report -name "*.json" | head -n 1') + +# Ensure the report file was found +if [ -z "$REPORT_PATH_IN_VOLUME" ]; then + echo "Conformance report not found in the volume." + exit 1 +fi + +# Copy the conformance report from the volume to the specified output path +docker cp "$TEMP_CONTAINER_NAME:$REPORT_PATH_IN_VOLUME" "$OUTPUT_PATH"