diff --git a/.github/workflows/nightly-backward-compatibility.yml b/.github/workflows/nightly-backward-compatibility.yml index 586f300a040..47f2003274e 100644 --- a/.github/workflows/nightly-backward-compatibility.yml +++ b/.github/workflows/nightly-backward-compatibility.yml @@ -1,9 +1,10 @@ name: Nightly backward compatibility on: + # Important note about scheduled workflows: + # Notifications for scheduled workflows are sent to the user who last modified the cron syntax in the workflow file. schedule: - - cron: "0 2 * * *" - - cron: "0 14 * * *" + - cron: "30 2 * * *" workflow_dispatch: inputs: total-releases: @@ -15,11 +16,30 @@ on: description: "Cardano node version used in e2e" required: true type: string - default: "10.1.1" + default: "10.1.2" jobs: + prepare-env-variables: + runs-on: ubuntu-22.04 + outputs: + total_releases: ${{ steps.set-env.outputs.total_releases }} + cardano_node_version: ${{ steps.set-env.outputs.cardano_node_version }} + steps: + - name: Prepare env variables + id: set-env + shell: bash + run: | + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "total_releases=3" >> $GITHUB_OUTPUT + echo 'cardano_node_version=["10.1.2"]' >> $GITHUB_OUTPUT + else + echo "total_releases=${{ inputs.total-releases }}" >> $GITHUB_OUTPUT + echo "cardano_node_version=[\"${{ inputs.cardano-node-version }}\"]" >> $GITHUB_OUTPUT + fi + prepare-binaries: runs-on: ubuntu-22.04 + needs: [prepare-env-variables] outputs: tags: ${{ steps.tags-test-lab.outputs.tags }} steps: @@ -27,10 +47,19 @@ jobs: uses: actions/checkout@v4 - name: Download releases artifacts binaries + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash run: | - ./.github/workflows/scripts/download-distribution-binaries.sh ${{ inputs.total-releases } + ./.github/workflows/scripts/download-distribution-binaries.sh ${{ needs.prepare-env-variables.outputs.total_releases }} + + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable - name: Build e2e + shell: bash run: | cargo build --release --bin mithril-end-to-end cp ./target/release/mithril-end-to-end ./mithril-binaries/unstable @@ -43,21 +72,21 @@ jobs: - name: Prepare test lab tags id: tags-test-lab + shell: bash run: | - TAGS=$(cat ./mithril-binaries/tags.json) + TAGS=$(jq -c '.' ./mithril-binaries/tags.json) echo "Test Lab Tags: $TAGS" echo "tags=$TAGS" >> $GITHUB_OUTPUT e2e: runs-on: ubuntu-22.04 - needs: [prepare-binaries] + needs: [prepare-env-variables, prepare-binaries] strategy: fail-fast: false matrix: tag: ${{ fromJSON(needs.prepare-binaries.outputs.tags) }} - node: - [mithril-aggregator, mithril-client, mithril-signer, mithril-relay] - cardano_node_version: ${{ inputs.cardano-node-version }} + node: [mithril-aggregator, mithril-client, mithril-signer] + cardano_node_version: ${{ fromJSON(needs.prepare-env-variables.outputs.cardano_node_version) }} run_id: ["#1"] steps: @@ -71,6 +100,7 @@ jobs: path: ./mithril-binaries - name: Prepare binaries + shell: bash run: | mkdir -p mithril-binaries/e2e cp ./mithril-binaries/unstable/* ./mithril-binaries/e2e @@ -84,9 +114,118 @@ jobs: mkdir artifacts - name: Run E2E tests + shell: bash run: | - ./mithril-binaries/e2e/mithril-end-to-end -vvv \\ - --bin-directory ./mithril-binaries/e2e \\ - --work-directory=./artifacts \\ - --devnet-scripts-directory=./mithril-test-lab/mithril-devnet \\ - --cardano-node-version ${{ matrix.cardano_node_version }} + ./mithril-binaries/e2e/mithril-end-to-end -vvv \ + --bin-directory ./mithril-binaries/e2e \ + --work-directory=./artifacts \ + --devnet-scripts-directory=./mithril-test-lab/mithril-devnet \ + --cardano-node-version ${{ matrix.cardano_node_version }} \ + --cardano-slot-length 0.25 \ + --cardano-epoch-length 45.0 \ + && echo "SUCCESS=true" >> $GITHUB_ENV \ + || (echo "SUCCESS=false" >> $GITHUB_ENV && exit 1) + + - name: Define the JSON file name for the test result + shell: bash + if: success() || failure() + run: echo "RESULT_FILE_NAME=e2e-test-result-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-tag_${{ matrix.tag }}-node-${{ matrix.node }}-cardano-${{ matrix.cardano_node_version }}-run_id_${{ matrix.run_id }}" >> $GITHUB_ENV + + - name: Write test result JSON + if: success() || failure() + shell: bash + run: | + AGGREGATOR_TAG="unstable" + SIGNER_TAG="unstable" + CLIENT_TAG="unstable" + + case "$NODE" in + mithril-aggregator) + AGGREGATOR_TAG="${{ matrix.tag }}" + ;; + mithril-signer) + SIGNER_TAG="${{ matrix.tag }}" + ;; + mithril-client) + CLIENT_TAG="${{ matrix.tag }}" + ;; + esac + + jq -n --arg TAG "${{ matrix.tag }}" \ + --arg NODE "${{ matrix.node }}" \ + --arg CARDANO_NODE "${{ matrix.cardano_node_version }}" \ + --arg AGGREGATOR "$AGGREGATOR_TAG" \ + --arg SIGNER "$SIGNER_TAG" \ + --arg CLIENT "$CLIENT_TAG" \ + --argjson SUCCESS "${{ env.SUCCESS }}" \ + '{tag: $TAG, node: $NODE, mithril_signer: $SIGNER, mithril_aggregator: $AGGREGATOR, mithril_client: $CLIENT, cardano_node_version: $CARDANO_NODE, success: $SUCCESS}' \ + > ./${{ env.RESULT_FILE_NAME }}.json + + - name: Upload test result JSON + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.RESULT_FILE_NAME }} + path: ./${{ env.RESULT_FILE_NAME }}.json + + - name: Upload E2E Tests Artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: mithril-e2e-tests-artifacts-run_${{ github.run_number }}-attempt_${{ github.run_attempt }}-tag_${{ matrix.tag }}-node-${{ matrix.node }}-cardano-${{ matrix.cardano_node_version }}-run_id_${{ matrix.run_id }} + path: | + ./artifacts/* + # including node.sock makes the upload fails so exclude them: + !./artifacts/**/node.sock + # exclude cardano tools, saving ~50mb of data: + !./artifacts/devnet/cardano-cli + !./artifacts/devnet/cardano-node + if-no-files-found: error + + summarize-test-results: + runs-on: ubuntu-22.04 + needs: [e2e] + if: success() || failure() + + steps: + - name: Download all test result artifacts + uses: actions/download-artifact@v4 + with: + path: ./test-results + pattern: e2e-test-result* + merge-multiple: true + + - name: Concatenate JSON result files into summary.json + shell: bash + run: | + jq -s '.' ./test-results/e2e-test-result-*.json > ./test-results/summary.json + + - name: Add distributions backward compatibility summary + shell: bash + run: | + CHECK_MARK=":heavy_check_mark:" + CROSS_MARK=":no_entry:" + + echo "## Distributions backward compatibility" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "This is the compatibility report of previous distributions nodes with the current unstable nodes." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "| Compatibility | mithril-signer | mithril-aggregator | mithril-client |" >> $GITHUB_STEP_SUMMARY + echo "| --- | :---: | :---: | :---: |" >> $GITHUB_STEP_SUMMARY + + # Transform summary.json into Markdown table rows + jq -r --arg CHECK_MARK "$CHECK_MARK" --arg CROSS_MARK "$CROSS_MARK" \ + 'group_by(.tag) | + sort_by(.[0].tag | tonumber) | reverse | + .[] | + { + tag: .[0].tag, + signer: (map(select(.node == "mithril-signer") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")), + aggregator: (map(select(.node == "mithril-aggregator") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")), + client: (map(select(.node == "mithril-client") | if .success then $CHECK_MARK else $CROSS_MARK end) | join("")) + } | + "| `\(.tag)` | \(.signer) | \(.aggregator) | \(.client) |"' "./test-results/summary.json" >> $GITHUB_STEP_SUMMARY + + cat "$GITHUB_STEP_SUMMARY"