diff --git a/.github/scripts/process-differential-tests-report.py b/.github/scripts/process-differential-tests-report.py new file mode 100644 index 0000000000000..35258d508cbd0 --- /dev/null +++ b/.github/scripts/process-differential-tests-report.py @@ -0,0 +1,241 @@ +""" +This script is used to turn the JSON report produced by the revive differential tests tool into an +easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK +CI. The full models used in the JSON report can be found in the revive differential tests repo and +the models used in this script are just a partial reproduction of the full report models. +""" + +import json, typing, io, sys + + +class Report(typing.TypedDict): + context: "Context" + execution_information: dict["MetadataFilePathString", "MetadataFileReport"] + + +class MetadataFileReport(typing.TypedDict): + case_reports: dict["CaseIdxString", "CaseReport"] + + +class CaseReport(typing.TypedDict): + mode_execution_reports: dict["ModeString", "ExecutionReport"] + + +class ExecutionReport(typing.TypedDict): + status: "TestCaseStatus" + + +class Context(typing.TypedDict): + Test: "TestContext" + + +class TestContext(typing.TypedDict): + corpus_configuration: "CorpusConfiguration" + + +class CorpusConfiguration(typing.TypedDict): + test_specifiers: list["TestSpecifier"] + + +class CaseStatusSuccess(typing.TypedDict): + status: typing.Literal["Succeeded"] + steps_executed: int + + +class CaseStatusFailure(typing.TypedDict): + status: typing.Literal["Failed"] + reason: str + + +class CaseStatusIgnored(typing.TypedDict): + status: typing.Literal["Ignored"] + reason: str + + +TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] +"""A union type of all of the possible statuses that could be reported for a case.""" + +TestSpecifier = str +"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" + +ModeString = str +"""The mode string. For example Y+ >=0.8.13""" + +MetadataFilePathString = str +"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" + +CaseIdxString = str +"""The index of a case as a string. For example '0'""" + + +def path_relative_to_resolc_compiler_test_directory(path: str) -> str: + """ + Given a path, this function returns the path relative to the resolc-compiler-test directory. The + following is an example of an input and an output: + + Input: ~/polkadot-sdk/revive-differential-tests/resolc-compiler-tests/fixtures/solidity/test.json + Output: test.json + """ + + return f"{path.split('resolc-compiler-tests/fixtures/solidity')[-1].strip('/')}" + + +def main() -> None: + with open(sys.argv[1], "r") as file: + report: Report = json.load(file) + + # Starting the markdown document and adding information to it as we go. + markdown_document: io.TextIOWrapper = open("report.md", "w") + print("# Differential Tests Results", file=markdown_document) + + # Getting all of the test specifiers from the report and making them relative to the tests dir. + test_specifiers: list[str] = list( + map( + path_relative_to_resolc_compiler_test_directory, + report["context"]["Test"]["corpus_configuration"]["test_specifiers"], + ) + ) + print("## Specified Tests", file=markdown_document) + for test_specifier in test_specifiers: + print(f"* `{test_specifier}`", file=markdown_document) + + # Counting the total number of test cases, successes, failures, and ignored tests + total_number_of_cases: int = 0 + total_number_of_successes: int = 0 + total_number_of_failures: int = 0 + total_number_of_ignores: int = 0 + for _, mode_to_case_mapping in report["execution_information"].items(): + for _, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for _, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] + + total_number_of_cases += 1 + if status["status"] == "Succeeded": + total_number_of_successes += 1 + elif status["status"] == "Failed": + total_number_of_failures += 1 + elif status["status"] == "Ignored": + total_number_of_ignores += 1 + else: + raise Exception( + f"Encountered a status that's unknown to the script: {status}" + ) + + print("## Counts", file=markdown_document) + print( + f"* **Total Number of Test Cases:** {total_number_of_cases}", + file=markdown_document, + ) + print( + f"* **Total Number of Successes:** {total_number_of_successes}", + file=markdown_document, + ) + print( + f"* **Total Number of Failures:** {total_number_of_failures}", + file=markdown_document, + ) + print( + f"* **Total Number of Ignores:** {total_number_of_ignores}", + file=markdown_document, + ) + + # Grouping the various test cases into dictionaries and groups depending on their status to make + # them easier to include in the markdown document later on. + successful_cases: dict[ + MetadataFilePathString, dict[CaseIdxString, set[ModeString]] + ] = {} + for metadata_file_path, mode_to_case_mapping in report[ + "execution_information" + ].items(): + for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for mode_string, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] + metadata_file_path: str = ( + path_relative_to_resolc_compiler_test_directory(metadata_file_path) + ) + mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-") + + if status["status"] == "Succeeded": + successful_cases.setdefault( + metadata_file_path, + {}, + ).setdefault( + case_idx_string, set() + ).add(mode_string) + + print("## Failures", file=markdown_document) + print( + "The test specifiers seen in this section have the format 'path::case_idx::compilation_mode'\ + and they're compatible with the revive differential tests framework and can be specified\ + to it directly in the same way that they're provided through the `--test` argument of the\ + framework.\n", + file=markdown_document, + ) + print( + "The failures are provided in an expandable section to ensure that the PR does not get \ + polluted with information. Please click on the section below for more information", + file=markdown_document, + ) + print( + "
Detailed Differential Tests Failure Information\n\n", + file=markdown_document, + ) + print("| Test Specifier | Failure Reason | Note |", file=markdown_document) + print("| -- | -- | -- |", file=markdown_document) + + for metadata_file_path, mode_to_case_mapping in report[ + "execution_information" + ].items(): + for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for mode_string, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] + metadata_file_path: str = ( + path_relative_to_resolc_compiler_test_directory(metadata_file_path) + ) + mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-") + + if status["status"] != "Failed": + continue + + failure_reason: str = status["reason"].replace("\n", " ") + + note: str = "" + modes_where_this_case_succeeded: set[ModeString] = ( + successful_cases.setdefault( + metadata_file_path, + {}, + ).setdefault(case_idx_string, set()) + ) + if len(modes_where_this_case_succeeded) != 0: + note: str = ( + f"This test case succeeded with other compilation modes: {modes_where_this_case_succeeded}" + ) + + test_specifier: str = ( + f"{metadata_file_path}::{case_idx_string}::{mode_string}" + ) + print( + f"| `{test_specifier}` | `{failure_reason}` | {note} |", + file=markdown_document, + ) + print("\n\n
", file=markdown_document) + + # The primary downside of not using `with`, but I guess it's better since I don't want to over + # indent the code. + markdown_document.close() + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 9dcafac106d60..b193e060bfe56 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -18,6 +18,62 @@ jobs: needs: isdraft uses: ./.github/workflows/reusable-preflight.yml + differential-tests-revive-dev-node-revm: + needs: [preflight] + runs-on: ${{ needs.preflight.outputs.RUNNER }} + if: ${{ needs.preflight.outputs.changes_rust }} + timeout-minutes: 60 + container: + image: ${{ needs.preflight.outputs.IMAGE }} + permissions: + pull-requests: write + steps: + - name: Checkout the Polkadot SDK + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Update the Installed Python + run: apt-get update && apt-get install -y python3-pip python3 + - name: Building the dependencies from the Polkadot SDK + run: forklift cargo build --locked --profile release -p pallet-revive-eth-rpc -p revive-dev-node + - name: Checkout the Differential Tests Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: paritytech/revive-differential-tests + ref: 75159229dff6e3cc0b1837c95fb053f7bb2a7cb5 + path: revive-differential-tests + submodules: recursive + - name: Installing Retester + run: forklift cargo install --locked --path revive-differential-tests/crates/core + - name: Creating a workdir for retester + run: mkdir workdir + - name: Downloading & Initializing the compilation caches + run: | + curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/paritytech/revive-differential-tests/releases/download/compilation-caches-v1.0/cache.tar.gz" + tar -zxf cache.tar.gz -C ./workdir + - name: Running the Differential Tests + run: | + retester test \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/simple \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/complex \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \ + --platform revive-dev-node-revm-solc \ + --concurrency.number-of-nodes 10 \ + --concurrency.number-of-threads 10 \ + --concurrency.number-of-concurrent-tasks 50 \ + --working-directory ./workdir \ + --revive-dev-node.consensus manual-seal-200 \ + --revive-dev-node.path ./target/release/revive-dev-node \ + --eth-rpc.path ./target/release/eth-rpc + - name: Creating a markdown report of the test execution + run: | + mv ./workdir/*.json report.json + python3 ./.github/scripts/process-differential-tests-report.py report.json + - name: Posting the report as a comment on the PR + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 + if: ${{ contains(github.event.pull_request.labels.*.name, 'T7-smart_contracts') }} + with: + header: diff-tests-report + path: report.md + evm-test-suite: needs: [preflight] runs-on: ${{ needs.preflight.outputs.RUNNER }} @@ -84,4 +140,4 @@ jobs: exit 1 else echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY - fi \ No newline at end of file + fi diff --git a/prdoc/pr_10071.prdoc b/prdoc/pr_10071.prdoc new file mode 100644 index 0000000000000..e9d5a34336ccf --- /dev/null +++ b/prdoc/pr_10071.prdoc @@ -0,0 +1,8 @@ +title: Adding Retester to CI +doc: +- audience: Runtime Dev + description: |- + # Description + + This PR adds differential tests as part of the CI of the polkadot SDK. Currently, a job will be started when pushing to master or when a PR is opened that runs the [differential testing framework](https://github.com/paritytech/revive-differential-tests) with the `revive-dev-node-revm-resolc` target. +crates: []