Adding Retester to CI

.github/scripts/process-differential-tests-report.py

@@ -0,0 +1,241 @@
+"""
+This script is used to turn the JSON report produced by the revive differential tests tool into an
+easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK
+CI. The full models used in the JSON report can be found in the revive differential tests repo and
+the models used in this script are just a partial reproduction of the full report models.
+"""
+
+import json, typing, io, sys
+
+
+class Report(typing.TypedDict):
+    context: "Context"
+    execution_information: dict["MetadataFilePathString", "MetadataFileReport"]
+
+
+class MetadataFileReport(typing.TypedDict):
+    case_reports: dict["CaseIdxString", "CaseReport"]
+
+
+class CaseReport(typing.TypedDict):
+    mode_execution_reports: dict["ModeString", "ExecutionReport"]
+
+
+class ExecutionReport(typing.TypedDict):
+    status: "TestCaseStatus"
+
+
+class Context(typing.TypedDict):
+    Test: "TestContext"
+
+
+class TestContext(typing.TypedDict):
+    corpus_configuration: "CorpusConfiguration"
+
+
+class CorpusConfiguration(typing.TypedDict):
+    test_specifiers: list["TestSpecifier"]
+
+
+class CaseStatusSuccess(typing.TypedDict):
+    status: typing.Literal["Succeeded"]
+    steps_executed: int
+
+
+class CaseStatusFailure(typing.TypedDict):
+    status: typing.Literal["Failed"]
+    reason: str
+
+
+class CaseStatusIgnored(typing.TypedDict):
+    status: typing.Literal["Ignored"]
+    reason: str
+
+
+TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored]
+"""A union type of all of the possible statuses that could be reported for a case."""
+
+TestSpecifier = str
+"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+"""
+
+ModeString = str
+"""The mode string. For example Y+ >=0.8.13"""
+
+MetadataFilePathString = str
+"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json"""
+
+CaseIdxString = str
+"""The index of a case as a string. For example '0'"""
+
+
+def path_relative_to_resolc_compiler_test_directory(path: str) -> str:
+    """
+    Given a path, this function returns the path relative to the resolc-compiler-test directory. The
+    following is an example of an input and an output:
+
+    Input: ~/polkadot-sdk/revive-differential-tests/resolc-compiler-tests/fixtures/solidity/test.json
+    Output: test.json
+    """
+
+    return f"{path.split('resolc-compiler-tests/fixtures/solidity')[-1].strip('/')}"
+
+
+def main() -> None:
+    with open(sys.argv[1], "r") as file:
+        report: Report = json.load(file)
+
+    # Starting the markdown document and adding information to it as we go.
+    markdown_document: io.TextIOWrapper = open("report.md", "w")
+    print("# Differential Tests Results", file=markdown_document)
+
+    # Getting all of the test specifiers from the report and making them relative to the tests dir.
+    test_specifiers: list[str] = list(
+        map(
+            path_relative_to_resolc_compiler_test_directory,
+            report["context"]["Test"]["corpus_configuration"]["test_specifiers"],
+        )
+    )
+    print("## Specified Tests", file=markdown_document)
+    for test_specifier in test_specifiers:
+        print(f"* `{test_specifier}`", file=markdown_document)
+
+    # Counting the total number of test cases, successes, failures, and ignored tests
+    total_number_of_cases: int = 0
+    total_number_of_successes: int = 0
+    total_number_of_failures: int = 0
+    total_number_of_ignores: int = 0
+    for _, mode_to_case_mapping in report["execution_information"].items():
+        for _, case_idx_to_report_mapping in mode_to_case_mapping[
+            "case_reports"
+        ].items():
+            for _, execution_report in case_idx_to_report_mapping[
+                "mode_execution_reports"
+            ].items():
+                status: TestCaseStatus = execution_report["status"]
+
+                total_number_of_cases += 1
+                if status["status"] == "Succeeded":
+                    total_number_of_successes += 1
+                elif status["status"] == "Failed":
+                    total_number_of_failures += 1
+                elif status["status"] == "Ignored":
+                    total_number_of_ignores += 1
+                else:
+                    raise Exception(
+                        f"Encountered a status that's unknown to the script: {status}"
+                    )
+
+    print("## Counts", file=markdown_document)
+    print(
+        f"* **Total Number of Test Cases:** {total_number_of_cases}",
+        file=markdown_document,
+    )
+    print(
+        f"* **Total Number of Successes:** {total_number_of_successes}",
+        file=markdown_document,
+    )
+    print(
+        f"* **Total Number of Failures:** {total_number_of_failures}",
+        file=markdown_document,
+    )
+    print(
+        f"* **Total Number of Ignores:** {total_number_of_ignores}",
+        file=markdown_document,
+    )
+
+    # Grouping the various test cases into dictionaries and groups depending on their status to make
+    # them easier to include in the markdown document later on.
+    successful_cases: dict[
+        MetadataFilePathString, dict[CaseIdxString, set[ModeString]]
+    ] = {}
+    for metadata_file_path, mode_to_case_mapping in report[
+        "execution_information"
+    ].items():
+        for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[
+            "case_reports"
+        ].items():
+            for mode_string, execution_report in case_idx_to_report_mapping[
+                "mode_execution_reports"
+            ].items():
+                status: TestCaseStatus = execution_report["status"]
+                metadata_file_path: str = (
+                    path_relative_to_resolc_compiler_test_directory(metadata_file_path)
+                )
+                mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-")
+
+                if status["status"] == "Succeeded":
+                    successful_cases.setdefault(
+                        metadata_file_path,
+                        {},
+                    ).setdefault(
+                        case_idx_string, set()
+                    ).add(mode_string)
+
+    print("## Failures", file=markdown_document)
+    print(
+        "The test specifiers seen in this section have the format 'path::case_idx::compilation_mode'\
+          and they're compatible with the revive differential tests framework and can be specified\
+          to it directly in the same way that they're provided through the `--test` argument of the\
+          framework.\n",
+        file=markdown_document,
+    )
+    print(
+        "The failures are provided in an expandable section to ensure that the PR does not get \
+        polluted with information. Please click on the section below for more information",
+        file=markdown_document,
+    )
+    print(
+        "<details><summary>Detailed Differential Tests Failure Information</summary>\n\n",
+        file=markdown_document,
+    )
+    print("| Test Specifier | Failure Reason | Note |", file=markdown_document)
+    print("| -- | -- | -- |", file=markdown_document)
+
+    for metadata_file_path, mode_to_case_mapping in report[
+        "execution_information"
+    ].items():
+        for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[
+            "case_reports"
+        ].items():
+            for mode_string, execution_report in case_idx_to_report_mapping[
+                "mode_execution_reports"
+            ].items():
+                status: TestCaseStatus = execution_report["status"]
+                metadata_file_path: str = (
+                    path_relative_to_resolc_compiler_test_directory(metadata_file_path)
+                )
+                mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-")
+
+                if status["status"] != "Failed":
+                    continue
+
+                failure_reason: str = status["reason"].replace("\n", " ")
+
+                note: str = ""
+                modes_where_this_case_succeeded: set[ModeString] = (
+                    successful_cases.setdefault(
+                        metadata_file_path,
+                        {},
+                    ).setdefault(case_idx_string, set())
+                )
+                if len(modes_where_this_case_succeeded) != 0:
+                    note: str = (
+                        f"This test case succeeded with other compilation modes: {modes_where_this_case_succeeded}"
+                    )
+
+                test_specifier: str = (
+                    f"{metadata_file_path}::{case_idx_string}::{mode_string}"
+                )
+                print(
+                    f"| `{test_specifier}` | `{failure_reason}` | {note} |",
+                    file=markdown_document,
+                )
+    print("\n\n</details>", file=markdown_document)
+
+    # The primary downside of not using `with`, but I guess it's better since I don't want to over
+    # indent the code.
+    markdown_document.close()
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/tests-evm.yml

@@ -18,6 +18,62 @@ jobs:
     needs: isdraft
     uses: ./.github/workflows/reusable-preflight.yml
 
+  differential-tests-revive-dev-node-revm:
+    needs: [preflight]
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
+    timeout-minutes: 60
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout the Polkadot SDK
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Update the Installed Python
+        run: apt-get update && apt-get install -y python3-pip python3
+      - name: Building the dependencies from the Polkadot SDK
+        run: forklift cargo build --locked --profile release -p pallet-revive-eth-rpc -p revive-dev-node
+      - name: Checkout the Differential Tests Repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          repository: paritytech/revive-differential-tests
+          ref: 75159229dff6e3cc0b1837c95fb053f7bb2a7cb5
+          path: revive-differential-tests
+          submodules: recursive
+      - name: Installing Retester
+        run: forklift cargo install --locked --path revive-differential-tests/crates/core
+      - name: Creating a workdir for retester
+        run: mkdir workdir
+      - name: Downloading & Initializing the compilation caches
+        run: |
+          curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/paritytech/revive-differential-tests/releases/download/compilation-caches-v1.0/cache.tar.gz"
+          tar -zxf cache.tar.gz -C ./workdir
+      - name: Running the Differential Tests
+        run: |
+          retester test \
+            --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/simple \
+            --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/complex \
+            --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \
+            --platform revive-dev-node-revm-solc \
+            --concurrency.number-of-nodes 10 \
+            --concurrency.number-of-threads 10 \
+            --concurrency.number-of-concurrent-tasks 50 \
+            --working-directory ./workdir \
+            --revive-dev-node.consensus manual-seal-200 \
+            --revive-dev-node.path ./target/release/revive-dev-node \
+            --eth-rpc.path ./target/release/eth-rpc
+      - name: Creating a markdown report of the test execution
+        run: |
+          mv ./workdir/*.json report.json
+          python3 ./.github/scripts/process-differential-tests-report.py report.json
+      - name: Posting the report as a comment on the PR
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'T7-smart_contracts') }}
+        with:
+          header: diff-tests-report
+          path: report.md
+
   evm-test-suite:
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
@@ -84,4 +140,4 @@ jobs:
             exit 1
           else
             echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY
-          fi
+          fi

prdoc/pr_10071.prdoc

@@ -0,0 +1,8 @@
+title: Adding Retester to CI
+doc:
+- audience: Runtime Dev
+  description: |-
+    # Description
+
+    This PR adds differential tests as part of the CI of the polkadot SDK. Currently, a job will be started when pushing to master or when a PR is opened that runs the [differential testing framework](https://github.com/paritytech/revive-differential-tests) with the `revive-dev-node-revm-resolc` target.
+crates: []