From bcb9b341103656ae96f58744ce88020892f50930 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Tue, 21 Oct 2025 17:46:51 +0300 Subject: [PATCH 01/24] Initial setup of retester in CI --- .github/workflows/tests-evm.yml | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 54c7a821418af..eb32e02992434 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -18,6 +18,47 @@ jobs: needs: isdraft uses: ./.github/workflows/reusable-preflight.yml + differential-tests-revive-dev-node-revm: + needs: [preflight] + runs-on: ${{ needs.preflight.outputs.RUNNER }} + if: ${{ needs.preflight.outputs.changes_rust }} + timeout-minutes: 60 + container: + image: ${{ needs.preflight.outputs.IMAGE }} + steps: + - name: Checkout the Polkadot SDK + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Building the dependencies from the Polkadot SDK + run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node + - name: Checkout the Differential Tests Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: paritytech/revive-differential-tests + ref: 48e7d691583d931fdfd6b647dc006501785c22b0 + path: revive-differential-tests + submodules: recursive + - name: Installing Retester + run: forklift cargo install --locked --path revive-differential-tests/crates/core + - name: Creating a workdir for retester + run: mkdir workdir + - name: Downloading the compilation caches + run: curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/0xOmarA/revive-differential-tests-compiler-cache/releases/download/v1.0.0/cache.tar.gz" + - name: Decompressing the compilation caches + run: tar -zxf cache.tar.gz -C ./workdir + - name: Running the Differential Tests + run: | + retester test \ + --test ./resolc-compiler-tests/fixtures/solidity/simple \ + --test ./resolc-compiler-tests/fixtures/solidity/complex \ + --test ./resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \ + --platform revive-dev-node-revm-solc \ + --concurrency.number-of-nodes 10 \ + --concurrency.number-of-threads 10 \ + --concurrency.number-of-concurrent-tasks 50 \ + --working-directory ./workdir \ + --revive-dev-node.path ./target/production/revive-dev-node \ + --eth-rpc.path ./target/production/eth-rpc + evm-test-suite: needs: [preflight] runs-on: ${{ needs.preflight.outputs.RUNNER }} @@ -91,4 +132,4 @@ jobs: exit 1 else echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY - fi \ No newline at end of file + fi From 338b1625b41088b79e00161d3176ea8e4ca8ef2e Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 00:41:34 +0300 Subject: [PATCH 02/24] Update the path of the tests --- .github/workflows/tests-evm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index eb32e02992434..44bb408e336a2 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -48,9 +48,9 @@ jobs: - name: Running the Differential Tests run: | retester test \ - --test ./resolc-compiler-tests/fixtures/solidity/simple \ - --test ./resolc-compiler-tests/fixtures/solidity/complex \ - --test ./resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/simple \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/complex \ + --test ./revive-differential-tests/resolc-compiler-tests/fixtures/solidity/translated_semantic_tests \ --platform revive-dev-node-revm-solc \ --concurrency.number-of-nodes 10 \ --concurrency.number-of-threads 10 \ From 18d8e0238839faae3b60a6908655249435ae6956 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 02:19:59 +0300 Subject: [PATCH 03/24] Allow for the generation of markdown reports --- .../process-differential-tests-report.py | 216 ++++++++++++++++++ .github/workflows/tests-evm.yml | 12 + 2 files changed, 228 insertions(+) create mode 100644 .github/scripts/process-differential-tests-report.py diff --git a/.github/scripts/process-differential-tests-report.py b/.github/scripts/process-differential-tests-report.py new file mode 100644 index 0000000000000..a29f39fcea1fa --- /dev/null +++ b/.github/scripts/process-differential-tests-report.py @@ -0,0 +1,216 @@ +""" +This script is used to turn the JSON report produced by the revive differential tests tool into an +easy to consume markdown document for the purpose of reporting this information in the Polkadot SDK +CI. The full models used in the JSON report can be found in the revive differential tests repo and +the models used in this script are just a partial reproduction of the full report models. +""" + +import json, typing, io + + +class Report(typing.TypedDict): + context: "Context" + test_case_information: dict[ + "MetadataFilePathString", + dict["ModeString", dict["CaseIdxString", "CaseReport"]], + ] + + +class Context(typing.TypedDict): + Test: "TestContext" + + +class TestContext(typing.TypedDict): + corpus_configuration: "CorpusConfiguration" + + +class CorpusConfiguration(typing.TypedDict): + test_specifiers: list["TestSpecifier"] + + +class CaseReport(typing.TypedDict): + status: "CaseStatus" + + +class CaseStatus(typing.TypedDict): + status: typing.Union[ + typing.Literal["Succeeded"], typing.Literal["Failed"], typing.Literal["Ignored"] + ] + reason: typing.Optional[str] + steps_executed: typing.Optional[int] + + +type TestSpecifier = str +"""A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" + +type ModeString = str +"""The mode string. For example Y+ >=0.8.13""" + +type MetadataFilePathString = str +"""The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" + +type CaseIdxString = str +"""The index of a case as a string. For example '0'""" + + +def path_relative_to_resolc_compiler_test_directory(path: str) -> str: + """ + Given a path, this function returns the path relative to the resolc-compiler-test directory. The + following is an example of an input and an output: + + Input: ~/polkadot-sdk/revive-differential-tests/resolc-compiler-tests/fixtures/solidity/test.json + Output: test.json + """ + + return f"{path.split('resolc-compiler-tests/fixtures/solidity')[-1].strip('/')}" + + +def main() -> None: + with open("report.json", "r") as file: + report: Report = json.load(file) + + # Starting the markdown document and adding information to it as we go. + markdown_document: io.TextIOWrapper = open("report.md", "w") + print("# Differential Tests Results", file=markdown_document) + + # Getting all of the test specifiers from the report and making them relative to the tests dir. + test_specifiers: list[str] = list( + map( + path_relative_to_resolc_compiler_test_directory, + report["context"]["Test"]["corpus_configuration"]["test_specifiers"], + ) + ) + print("## Specified Tests", file=markdown_document) + for test_specifier in test_specifiers: + print(f"* `{test_specifier}`", file=markdown_document) + + # Counting the total number of test cases, successes, failures, and ignored tests + total_number_of_cases: int = 0 + total_number_of_successes: int = 0 + total_number_of_failures: int = 0 + total_number_of_ignores: int = 0 + for _, mode_to_case_mapping in report["test_case_information"].items(): + for _, case_idx_to_report_mapping in mode_to_case_mapping.items(): + for _, case_report in case_idx_to_report_mapping.items(): + status: CaseStatus = case_report["status"] + + total_number_of_cases += 1 + if status["status"] == "Succeeded": + total_number_of_successes += 1 + elif status["status"] == "Failed": + total_number_of_failures += 1 + elif status["status"] == "Ignored": + total_number_of_ignores += 1 + else: + raise Exception( + f"Encountered a status that's unknown to the script: {status}" + ) + + print("## Counts", file=markdown_document) + print( + f"* **Total Number of Test Cases:** {total_number_of_cases}", + file=markdown_document, + ) + print( + f"* **Total Number of Successes:** {total_number_of_successes}", + file=markdown_document, + ) + print( + f"* **Total Number of Failures:** {total_number_of_failures}", + file=markdown_document, + ) + print( + f"* **Total Number of Ignores:** {total_number_of_ignores}", + file=markdown_document, + ) + + # Grouping the various test cases into dictionaries and groups depending on their status to make + # them easier to include in the markdown document later on. + successful_cases: dict[ + MetadataFilePathString, dict[CaseIdxString, set[ModeString]] + ] = {} + for metadata_file_path, mode_to_case_mapping in report[ + "test_case_information" + ].items(): + for mode_string, case_idx_to_report_mapping in mode_to_case_mapping.items(): + for case_idx_string, case_report in case_idx_to_report_mapping.items(): + status: CaseStatus = case_report["status"] + metadata_file_path: str = ( + path_relative_to_resolc_compiler_test_directory(metadata_file_path) + ) + mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-") + + if status["status"] == "Succeeded": + successful_cases.setdefault( + metadata_file_path, + {}, + ).setdefault( + case_idx_string, set() + ).add(mode_string) + + print("## Failures", file=markdown_document) + print( + "The test specifiers seen in this section have the format 'path::case_idx::compilation_mode'\ + and they're compatible with the revive differential tests framework and can be specified\ + to it directly in the same way that they're provided through the `--test` argument of the\ + framework.\n", + file=markdown_document, + ) + print( + "The failures are provided in an expandable section to ensure that the PR does not get \ + polluted with information. Please click on the section below for more information", + file=markdown_document, + ) + print( + "
Detailed Differential Tests Failure Information\n\n", + file=markdown_document, + ) + print("| Test Specifier | Failure Reason | Note |", file=markdown_document) + print("| -- | -- | -- |", file=markdown_document) + + for metadata_file_path, mode_to_case_mapping in report[ + "test_case_information" + ].items(): + for mode_string, case_idx_to_report_mapping in mode_to_case_mapping.items(): + for case_idx_string, case_report in case_idx_to_report_mapping.items(): + status: CaseStatus = case_report["status"] + metadata_file_path: str = ( + path_relative_to_resolc_compiler_test_directory(metadata_file_path) + ) + mode_string: str = mode_string.replace(" M3", "+").replace(" M0", "-") + + if status["status"] != "Failed": + continue + + failure_reason: str = typing.cast(str, status["reason"]).replace( + "\n", " " + ) + + note: str = "" + modes_where_this_case_succeeded: set[ModeString] = ( + successful_cases.setdefault( + metadata_file_path, + {}, + ).setdefault(case_idx_string, set()) + ) + if len(modes_where_this_case_succeeded) != 0: + note: str = ( + f"This test case succeeded with other compilation modes: {modes_where_this_case_succeeded}" + ) + + test_specifier: str = ( + f"{metadata_file_path}::{case_idx_string}::{mode_string}" + ) + print( + f"| `{test_specifier}` | `{failure_reason}` | {note} |", + file=markdown_document, + ) + print("\n\n
", file=markdown_document) + + # The primary downside of not using `with`, but I guess it's better since I don't want to over + # indent the code. + markdown_document.close() + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 44bb408e336a2..8f601deb75db8 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + pull-requests: write + jobs: isdraft: uses: ./.github/workflows/reusable-isdraft.yml @@ -58,6 +61,15 @@ jobs: --working-directory ./workdir \ --revive-dev-node.path ./target/production/revive-dev-node \ --eth-rpc.path ./target/production/eth-rpc + - name: Moving the report to the current directory + run: mv ./workdir/*.json report.json + - name: Creating a markdown report of the test execution + run: python3 ./scripts/process-differential-tests-report.py + - name: Posting the report as a comment on the PR + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: diff-tests-report + path: report.md evm-test-suite: needs: [preflight] From 4a7592f90bfa1bb5786ff240f55e7203b2278722 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 02:40:17 +0300 Subject: [PATCH 04/24] Update path of report processing script --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 8f601deb75db8..c2a9718ee598e 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -64,7 +64,7 @@ jobs: - name: Moving the report to the current directory run: mv ./workdir/*.json report.json - name: Creating a markdown report of the test execution - run: python3 ./scripts/process-differential-tests-report.py + run: python3 ./.github/scripts/process-differential-tests-report.py - name: Posting the report as a comment on the PR uses: marocchino/sticky-pull-request-comment@v2 with: From 3625142362886503f226e45193cfdafaa7cfd450 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 03:02:08 +0300 Subject: [PATCH 05/24] Update python prior running the script --- .github/workflows/tests-evm.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index c2a9718ee598e..6cd15400bc2d9 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -31,6 +31,10 @@ jobs: steps: - name: Checkout the Polkadot SDK uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Setting up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" - name: Building the dependencies from the Polkadot SDK run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node - name: Checkout the Differential Tests Repository From 37eabfa9c1a34eb77b97f2d8253416596a7212fb Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 03:08:37 +0300 Subject: [PATCH 06/24] Change the type aliasing syntax for older python --- .../process-differential-tests-report.py | 36 ++++++++++++------- .github/workflows/tests-evm.yml | 4 --- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/scripts/process-differential-tests-report.py b/.github/scripts/process-differential-tests-report.py index a29f39fcea1fa..730cf377f1394 100644 --- a/.github/scripts/process-differential-tests-report.py +++ b/.github/scripts/process-differential-tests-report.py @@ -32,24 +32,36 @@ class CaseReport(typing.TypedDict): status: "CaseStatus" -class CaseStatus(typing.TypedDict): - status: typing.Union[ - typing.Literal["Succeeded"], typing.Literal["Failed"], typing.Literal["Ignored"] - ] - reason: typing.Optional[str] - steps_executed: typing.Optional[int] +class CaseStatusSuccess(typing.TypedDict): + status: typing.Literal["Succeeded"] + steps_executed: int + + +class CaseStatusFailure(typing.TypedDict): + status: typing.Literal["Failed"] + reason: str + +class CaseStatusIgnored(typing.TypedDict): + status: typing.Literal["Ignored"] + reason: str -type TestSpecifier = str + +CaseStatus: typing.TypeAlias = typing.Union[ + CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored +] +"""A union type of all of the possible statuses that could be reported for a case.""" + +TestSpecifier: typing.TypeAlias = str """A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" -type ModeString = str +ModeString: typing.TypeAlias = str """The mode string. For example Y+ >=0.8.13""" -type MetadataFilePathString = str +MetadataFilePathString: typing.TypeAlias = str """The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" -type CaseIdxString = str +CaseIdxString: typing.TypeAlias = str """The index of a case as a string. For example '0'""" @@ -182,9 +194,7 @@ def main() -> None: if status["status"] != "Failed": continue - failure_reason: str = typing.cast(str, status["reason"]).replace( - "\n", " " - ) + failure_reason: str = status["reason"].replace("\n", " ") note: str = "" modes_where_this_case_succeeded: set[ModeString] = ( diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 6cd15400bc2d9..c2a9718ee598e 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -31,10 +31,6 @@ jobs: steps: - name: Checkout the Polkadot SDK uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Setting up Python - uses: actions/setup-python@v6 - with: - python-version: "3.13" - name: Building the dependencies from the Polkadot SDK run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node - name: Checkout the Differential Tests Repository From 734b1a82659585d687358a23787604d691bc1da6 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 03:24:22 +0300 Subject: [PATCH 07/24] Update the installed python --- .github/workflows/tests-evm.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index c2a9718ee598e..2b73b6aee3d40 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -31,6 +31,8 @@ jobs: steps: - name: Checkout the Polkadot SDK uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Update the Installed Python + run: sudo apt-get update && sudo apt-get install -y python3-pip python3 - name: Building the dependencies from the Polkadot SDK run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node - name: Checkout the Differential Tests Repository From 949b6577785dc5c00eb78d48c6877e5fb0683d35 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 03:31:33 +0300 Subject: [PATCH 08/24] Remove sudo from the python update command --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 2b73b6aee3d40..1856eaddb410c 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -32,7 +32,7 @@ jobs: - name: Checkout the Polkadot SDK uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Update the Installed Python - run: sudo apt-get update && sudo apt-get install -y python3-pip python3 + run: apt-get update && apt-get install -y python3-pip python3 - name: Building the dependencies from the Polkadot SDK run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node - name: Checkout the Differential Tests Repository From 0c7a930de7f1b4b56fe8091ec3c55c21273a88c4 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 03:47:01 +0300 Subject: [PATCH 09/24] Update the python alias syntax --- .github/scripts/process-differential-tests-report.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/scripts/process-differential-tests-report.py b/.github/scripts/process-differential-tests-report.py index 730cf377f1394..9e2914f0bbd3c 100644 --- a/.github/scripts/process-differential-tests-report.py +++ b/.github/scripts/process-differential-tests-report.py @@ -47,21 +47,19 @@ class CaseStatusIgnored(typing.TypedDict): reason: str -CaseStatus: typing.TypeAlias = typing.Union[ - CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored -] +CaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] """A union type of all of the possible statuses that could be reported for a case.""" -TestSpecifier: typing.TypeAlias = str +TestSpecifier = str """A test specifier string. For example resolc-compiler-tests/fixtures/solidity/test.json::0::Y+""" -ModeString: typing.TypeAlias = str +ModeString = str """The mode string. For example Y+ >=0.8.13""" -MetadataFilePathString: typing.TypeAlias = str +MetadataFilePathString = str """The path to a metadata file. For example resolc-compiler-tests/fixtures/solidity/test.json""" -CaseIdxString: typing.TypeAlias = str +CaseIdxString = str """The index of a case as a string. For example '0'""" From 7feb52edeb384571d1237c24c1b4f6822b51110f Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 10:58:12 +0300 Subject: [PATCH 10/24] Combine the compilation cache steps into one --- .github/workflows/tests-evm.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 1856eaddb410c..b36d3d7c9945d 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -46,10 +46,10 @@ jobs: run: forklift cargo install --locked --path revive-differential-tests/crates/core - name: Creating a workdir for retester run: mkdir workdir - - name: Downloading the compilation caches - run: curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/0xOmarA/revive-differential-tests-compiler-cache/releases/download/v1.0.0/cache.tar.gz" - - name: Decompressing the compilation caches - run: tar -zxf cache.tar.gz -C ./workdir + - name: Downloading & Initializing the compilation caches + run: | + curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/0xOmarA/revive-differential-tests-compiler-cache/releases/download/v1.0.0/cache.tar.gz" + tar -zxf cache.tar.gz -C ./workdir - name: Running the Differential Tests run: | retester test \ From e65ae97c4ef67f432eddbb0bc7b513d383d68e5a Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 10:59:16 +0300 Subject: [PATCH 11/24] Combine the report generation steps into one --- .github/workflows/tests-evm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index b36d3d7c9945d..07b8fc6c7b8b9 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -63,10 +63,10 @@ jobs: --working-directory ./workdir \ --revive-dev-node.path ./target/production/revive-dev-node \ --eth-rpc.path ./target/production/eth-rpc - - name: Moving the report to the current directory - run: mv ./workdir/*.json report.json - name: Creating a markdown report of the test execution - run: python3 ./.github/scripts/process-differential-tests-report.py + run: | + mv ./workdir/*.json report.json + python3 ./.github/scripts/process-differential-tests-report.py - name: Posting the report as a comment on the PR uses: marocchino/sticky-pull-request-comment@v2 with: From 087170b6afa93a177007c6c9b8ae8271374c4fbd Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 11:00:41 +0300 Subject: [PATCH 12/24] Remove un-needed step in the ci --- .github/workflows/tests-evm.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 07b8fc6c7b8b9..d9cc4595c897c 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -123,13 +123,6 @@ jobs: echo "== Running evm tests ==" START_REVIVE_DEV_NODE=true START_ETH_RPC=true deno task test:evm - - name: Collect tests results - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: evm-test-suite-${{ github.sha }} - path: evm-test-suite/test-logs/matter-labs-tests.log - confirm-required-test-evm-jobs-passed: runs-on: ubuntu-latest name: All test misc tests passed From 1c0fc20bce3c17cc3719419e83ea65b79d740967 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 11:35:03 +0300 Subject: [PATCH 13/24] Only generate report for smart contract PRs --- .github/workflows/tests-evm.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index d9cc4595c897c..563e0f4fa6562 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -39,7 +39,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: 48e7d691583d931fdfd6b647dc006501785c22b0 + ref: f1a911545ef0cfc1e1bfa5c91fc4d6194a273c52 path: revive-differential-tests submodules: recursive - name: Installing Retester @@ -69,6 +69,7 @@ jobs: python3 ./.github/scripts/process-differential-tests-report.py - name: Posting the report as a comment on the PR uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ contains(github.event.pull_request.labels.*.name, 'T7-smart_contracts') }} with: header: diff-tests-report path: report.md From 71b7c004cf2fceca98f1a558401d7084c286959a Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 11:53:05 +0300 Subject: [PATCH 14/24] Compile Polkadot SDK for release not production --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 563e0f4fa6562..0f0c96b6db7df 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -34,7 +34,7 @@ jobs: - name: Update the Installed Python run: apt-get update && apt-get install -y python3-pip python3 - name: Building the dependencies from the Polkadot SDK - run: forklift cargo build --locked --profile production -p pallet-revive-eth-rpc -p revive-dev-node + run: forklift cargo build --locked --profile release -p pallet-revive-eth-rpc -p revive-dev-node - name: Checkout the Differential Tests Repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: From 3db6478150ad5edc6cbc73414647eff92c1ada04 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 12:55:54 +0300 Subject: [PATCH 15/24] Compile Polkadot SDK for release not production --- .github/workflows/tests-evm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 0f0c96b6db7df..6f2b63ff38038 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -61,8 +61,8 @@ jobs: --concurrency.number-of-threads 10 \ --concurrency.number-of-concurrent-tasks 50 \ --working-directory ./workdir \ - --revive-dev-node.path ./target/production/revive-dev-node \ - --eth-rpc.path ./target/production/eth-rpc + --revive-dev-node.path ./target/release/revive-dev-node \ + --eth-rpc.path ./target/release/eth-rpc - name: Creating a markdown report of the test execution run: | mv ./workdir/*.json report.json From 9621977b8952f02cbed6856b646818ab7ca7df58 Mon Sep 17 00:00:00 2001 From: "cmd[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Oct 2025 10:14:59 +0000 Subject: [PATCH 16/24] Update from github-actions[bot] running command 'prdoc --audience runtime_dev' --- prdoc/pr_10071.prdoc | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 prdoc/pr_10071.prdoc diff --git a/prdoc/pr_10071.prdoc b/prdoc/pr_10071.prdoc new file mode 100644 index 0000000000000..e9d5a34336ccf --- /dev/null +++ b/prdoc/pr_10071.prdoc @@ -0,0 +1,8 @@ +title: Adding Retester to CI +doc: +- audience: Runtime Dev + description: |- + # Description + + This PR adds differential tests as part of the CI of the polkadot SDK. Currently, a job will be started when pushing to master or when a PR is opened that runs the [differential testing framework](https://github.com/paritytech/revive-differential-tests) with the `revive-dev-node-revm-resolc` target. +crates: [] From db1c6bdcbe1b02ab561f7663a2f761361995827c Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Wed, 22 Oct 2025 13:47:49 +0300 Subject: [PATCH 17/24] Scope permissions to the dt job --- .github/workflows/tests-evm.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 6f2b63ff38038..7a3932421c6d1 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -11,9 +11,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -permissions: - pull-requests: write - jobs: isdraft: uses: ./.github/workflows/reusable-isdraft.yml @@ -28,6 +25,8 @@ jobs: timeout-minutes: 60 container: image: ${{ needs.preflight.outputs.IMAGE }} + permissions: + pull-requests: write steps: - name: Checkout the Polkadot SDK uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 From a96a00178b0aafa0aaea0cc4c3fbc383080f852a Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Tue, 28 Oct 2025 02:05:45 +0300 Subject: [PATCH 18/24] Use commit hash for the action --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 7a3932421c6d1..7a30b1e1ff874 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -67,7 +67,7 @@ jobs: mv ./workdir/*.json report.json python3 ./.github/scripts/process-differential-tests-report.py - name: Posting the report as a comment on the PR - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 if: ${{ contains(github.event.pull_request.labels.*.name, 'T7-smart_contracts') }} with: header: diff-tests-report From 4d449951a752d774fdb91966042f03c1e56aa087 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Sat, 1 Nov 2025 06:22:33 +0300 Subject: [PATCH 19/24] Update for the latest report format --- .../process-differential-tests-report.py | 63 +++++++++------ .github/workflows/tests-evm.yml | 4 +- report.md | 78 +++++++++++++++++++ 3 files changed, 120 insertions(+), 25 deletions(-) create mode 100644 report.md diff --git a/.github/scripts/process-differential-tests-report.py b/.github/scripts/process-differential-tests-report.py index 9e2914f0bbd3c..35258d508cbd0 100644 --- a/.github/scripts/process-differential-tests-report.py +++ b/.github/scripts/process-differential-tests-report.py @@ -5,15 +5,24 @@ the models used in this script are just a partial reproduction of the full report models. """ -import json, typing, io +import json, typing, io, sys class Report(typing.TypedDict): context: "Context" - test_case_information: dict[ - "MetadataFilePathString", - dict["ModeString", dict["CaseIdxString", "CaseReport"]], - ] + execution_information: dict["MetadataFilePathString", "MetadataFileReport"] + + +class MetadataFileReport(typing.TypedDict): + case_reports: dict["CaseIdxString", "CaseReport"] + + +class CaseReport(typing.TypedDict): + mode_execution_reports: dict["ModeString", "ExecutionReport"] + + +class ExecutionReport(typing.TypedDict): + status: "TestCaseStatus" class Context(typing.TypedDict): @@ -28,10 +37,6 @@ class CorpusConfiguration(typing.TypedDict): test_specifiers: list["TestSpecifier"] -class CaseReport(typing.TypedDict): - status: "CaseStatus" - - class CaseStatusSuccess(typing.TypedDict): status: typing.Literal["Succeeded"] steps_executed: int @@ -47,7 +52,7 @@ class CaseStatusIgnored(typing.TypedDict): reason: str -CaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] +TestCaseStatus = typing.Union[CaseStatusSuccess, CaseStatusFailure, CaseStatusIgnored] """A union type of all of the possible statuses that could be reported for a case.""" TestSpecifier = str @@ -76,7 +81,7 @@ def path_relative_to_resolc_compiler_test_directory(path: str) -> str: def main() -> None: - with open("report.json", "r") as file: + with open(sys.argv[1], "r") as file: report: Report = json.load(file) # Starting the markdown document and adding information to it as we go. @@ -99,10 +104,14 @@ def main() -> None: total_number_of_successes: int = 0 total_number_of_failures: int = 0 total_number_of_ignores: int = 0 - for _, mode_to_case_mapping in report["test_case_information"].items(): - for _, case_idx_to_report_mapping in mode_to_case_mapping.items(): - for _, case_report in case_idx_to_report_mapping.items(): - status: CaseStatus = case_report["status"] + for _, mode_to_case_mapping in report["execution_information"].items(): + for _, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for _, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] total_number_of_cases += 1 if status["status"] == "Succeeded": @@ -140,11 +149,15 @@ def main() -> None: MetadataFilePathString, dict[CaseIdxString, set[ModeString]] ] = {} for metadata_file_path, mode_to_case_mapping in report[ - "test_case_information" + "execution_information" ].items(): - for mode_string, case_idx_to_report_mapping in mode_to_case_mapping.items(): - for case_idx_string, case_report in case_idx_to_report_mapping.items(): - status: CaseStatus = case_report["status"] + for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for mode_string, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] metadata_file_path: str = ( path_relative_to_resolc_compiler_test_directory(metadata_file_path) ) @@ -179,11 +192,15 @@ def main() -> None: print("| -- | -- | -- |", file=markdown_document) for metadata_file_path, mode_to_case_mapping in report[ - "test_case_information" + "execution_information" ].items(): - for mode_string, case_idx_to_report_mapping in mode_to_case_mapping.items(): - for case_idx_string, case_report in case_idx_to_report_mapping.items(): - status: CaseStatus = case_report["status"] + for case_idx_string, case_idx_to_report_mapping in mode_to_case_mapping[ + "case_reports" + ].items(): + for mode_string, execution_report in case_idx_to_report_mapping[ + "mode_execution_reports" + ].items(): + status: TestCaseStatus = execution_report["status"] metadata_file_path: str = ( path_relative_to_resolc_compiler_test_directory(metadata_file_path) ) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 4fade2a090d39..1104050ca76b6 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: f1a911545ef0cfc1e1bfa5c91fc4d6194a273c52 + ref: 33b5faca4529b659f427b80080bcae95bbb1bfd6 path: revive-differential-tests submodules: recursive - name: Installing Retester @@ -65,7 +65,7 @@ jobs: - name: Creating a markdown report of the test execution run: | mv ./workdir/*.json report.json - python3 ./.github/scripts/process-differential-tests-report.py + python3 ./.github/scripts/process-differential-tests-report.py report.json - name: Posting the report as a comment on the PR uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 if: ${{ contains(github.event.pull_request.labels.*.name, 'T7-smart_contracts') }} diff --git a/report.md b/report.md new file mode 100644 index 0000000000000..c15b054de5847 --- /dev/null +++ b/report.md @@ -0,0 +1,78 @@ +# Differential Tests Results +## Specified Tests +* `simple` +* `complex` +* `translated_semantic_tests` +## Counts +* **Total Number of Test Cases:** 26447 +* **Total Number of Successes:** 19140 +* **Total Number of Failures:** 56 +* **Total Number of Ignores:** 7251 +## Failures +The test specifiers seen in this section have the format 'path::case_idx::compilation_mode' and they're compatible with the revive differential tests framework and can be specified to it directly in the same way that they're provided through the `--test` argument of the framework. + +The failures are provided in an expandable section to ensure that the PR does not get polluted with information. Please click on the section below for more information +
Detailed Differential Tests Failure Information + + +| Test Specifier | Failure Reason | Note | +| -- | -- | -- | +| `complex/create/create2_many/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/create/create2_many/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/create/create_many/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'Y-', 'Y+', 'E+'} | +| `complex/create/create_many/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/create/create_many/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | +| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | +| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | +| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | +| `complex/defi/UniswapV3/test_evm.json::0::E+ =0.7.6` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/try_catch/create/error/test.json::11::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'E-', 'Y-', 'Y+'} | +| `complex/try_catch/create/panic/test.json::4::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'E-', 'Y-', 'Y+'} | +| `complex/voting/test.json::0::E+ >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/voting/test.json::0::E- >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/voting/test.json::0::Y+ >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/voting/test.json::0::Y- >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | +| `complex/yul_instructions/calldatasize/test_evm.json::11::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/yul_instructions/calldatasize/test_evm.json::11::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/yul_instructions/calldatasize/test_evm.json::11::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `complex/yul_instructions/calldatasize/test_evm.json::11::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `simple/context/difficulty.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/context/difficulty.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/context/difficulty.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/context/difficulty.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/try_catch/unbalanced_gas_limit.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | +| `simple/try_catch/unbalanced_gas_limit.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | +| `simple/try_catch/unbalanced_gas_limit.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | +| `simple/try_catch/unbalanced_gas_limit.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | +| `simple/yul_instructions/blockhash.sol::3::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'Y-'} | +| `simple/yul_instructions/blockhash.sol::3::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'Y-'} | +| `simple/yul_instructions/blockhash.sol::3::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0xc4ebbc3bb335eb93f63643b71d9ffce26155a64ac36d0179acb7dc71a0c94415` | This test case succeeded with other compilation modes: {'Y-'} | +| `simple/yul_instructions/blockhash.sol::4::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:255")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'E-', 'E+'} | +| `simple/yul_instructions/blockhash.sol::4::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:255")]) but got 0xf84672ca120abb8236e22bf8307ac96051277c332355cadfd08b5852b598a71a` | This test case succeeded with other compilation modes: {'E-', 'E+'} | +| `simple/yul_instructions/difficulty.sol::0::Y- =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/difficulty.sol::0::Y+ =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/difficulty.sol::0::E+ =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/difficulty.sol::0::E- =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/prevrandao.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/prevrandao.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/prevrandao.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `simple/yul_instructions/prevrandao.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | +| `translated_semantic_tests/array/array_storage_index_access/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/array/array_storage_length_access/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/array/array_storage_push_empty_length_address/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/array/array_storage_push_pop/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/array/copying/copy_removes_bytes_data/test.json::0::E-` | `Failed to execute all of the steps on the driver: Storage Empty Assertion Step Failed: server returned an error response: error code -32601: Method not found` | | +| `translated_semantic_tests/array/slices/array_slice_calldata_to_memory/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle the function call execution: Transaction inclusion watching timeout for 0xda5d8902f33d434b0e305acbed970dbadb0e1595cc7c3d1f097268ab95cab4aa: transaction was not confirmed within the timeout` | | +| `translated_semantic_tests/functionCall/call_options_overload/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Single(0x0000000000000000000000000000000000000000000000000de0b6b3a7640000) but got 0x0000000000000000000000000000000000000000000000000de0b6b3a754bdde` | | +| `translated_semantic_tests/functionCall/gas_and_value_basic/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected false but got true. Revert reason: None` | | +| `translated_semantic_tests/inheritance/inherited_function/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to deploy contracts for the function call step: Failed to get or deploy contract instance during input execution: Failed to deploy contract: Transaction inclusion watching timeout for 0x0d02f35e79908efcd422ce577a710f025fd31d0cab67ecc62f6ca3bb6fef0baf: transaction was not confirmed within the timeout` | | +| `translated_semantic_tests/inheritance/overloaded_function_call_with_if_else/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to deploy contracts for the function call step: Failed to get or deploy contract instance during input execution: Failed to deploy contract: Transaction inclusion watching timeout for 0x1aad59e22cffaafb040e8d96b7723f7cb0a7d09c9164218785831b4dba4ad74e: transaction was not confirmed within the timeout` | | +| `translated_semantic_tests/inlineAssembly/blobhash_index_exceeding_blob_count/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/libraries/external_call_with_storage_mapping_parameter/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/state/block_blobbasefee/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | +| `translated_semantic_tests/various/selfdestruct_post_cancun/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("panic: assertion failed (0x01)")` | | +| `translated_semantic_tests/various/selfdestruct_post_cancun_multiple_beneficiaries/test.json::0::E-` | `Failed to execute all of the steps on the driver: Balance Assertion Step Failed: Balance assertion failed - Expected 1000000000000000000 but got 0 for 0x1111111111111111111111111111111111111111 resolved to 0x1111111111111111111111111111111111111111` | | + + +
From 74b1abd78126e24fe234d934011bd7556a30b328 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Sat, 1 Nov 2025 06:42:55 +0300 Subject: [PATCH 20/24] Use manual seal for consensus in differential testing --- .github/workflows/tests-evm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 1104050ca76b6..2fcf956e3cdd3 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -60,6 +60,7 @@ jobs: --concurrency.number-of-threads 10 \ --concurrency.number-of-concurrent-tasks 50 \ --working-directory ./workdir \ + --revive-dev-node.consensus manual-seal-200 \ --revive-dev-node.path ./target/release/revive-dev-node \ --eth-rpc.path ./target/release/eth-rpc - name: Creating a markdown report of the test execution From 5d9222bb6abd05fefa26c5de6c664973d61d5c0f Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Sun, 2 Nov 2025 18:39:38 +0300 Subject: [PATCH 21/24] Remove accidentally comitted report --- report.md | 78 ------------------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 report.md diff --git a/report.md b/report.md deleted file mode 100644 index c15b054de5847..0000000000000 --- a/report.md +++ /dev/null @@ -1,78 +0,0 @@ -# Differential Tests Results -## Specified Tests -* `simple` -* `complex` -* `translated_semantic_tests` -## Counts -* **Total Number of Test Cases:** 26447 -* **Total Number of Successes:** 19140 -* **Total Number of Failures:** 56 -* **Total Number of Ignores:** 7251 -## Failures -The test specifiers seen in this section have the format 'path::case_idx::compilation_mode' and they're compatible with the revive differential tests framework and can be specified to it directly in the same way that they're provided through the `--test` argument of the framework. - -The failures are provided in an expandable section to ensure that the PR does not get polluted with information. Please click on the section below for more information -
Detailed Differential Tests Failure Information - - -| Test Specifier | Failure Reason | Note | -| -- | -- | -- | -| `complex/create/create2_many/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/create/create2_many/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/create/create_many/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'Y-', 'Y+', 'E+'} | -| `complex/create/create_many/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/create/create_many/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | -| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | -| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | -| `complex/defi/UniswapV2Router01/UniswapV2Pair/test.json::1::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("99999999999999900000000")]) but got 0x000000000000000000000000000000000000000000002a5a058fc295e70a1f00` | | -| `complex/defi/UniswapV3/test_evm.json::0::E+ =0.7.6` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/try_catch/create/error/test.json::11::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'E-', 'Y-', 'Y+'} | -| `complex/try_catch/create/panic/test.json::4::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | This test case succeeded with other compilation modes: {'E-', 'Y-', 'Y+'} | -| `complex/voting/test.json::0::E+ >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/voting/test.json::0::E- >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/voting/test.json::0::Y+ >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/voting/test.json::0::Y- >=0.8.1` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("")` | | -| `complex/yul_instructions/calldatasize/test_evm.json::11::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/yul_instructions/calldatasize/test_evm.json::11::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/yul_instructions/calldatasize/test_evm.json::11::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `complex/yul_instructions/calldatasize/test_evm.json::11::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `simple/context/difficulty.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/context/difficulty.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/context/difficulty.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/context/difficulty.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/try_catch/unbalanced_gas_limit.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | -| `simple/try_catch/unbalanced_gas_limit.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | -| `simple/try_catch/unbalanced_gas_limit.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | -| `simple/try_catch/unbalanced_gas_limit.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("0"), CalldataItem("1"), CalldataItem("0"), CalldataItem("0")]) but got 0x0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001` | | -| `simple/yul_instructions/blockhash.sol::3::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'Y-'} | -| `simple/yul_instructions/blockhash.sol::3::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'Y-'} | -| `simple/yul_instructions/blockhash.sol::3::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:256")]) but got 0xc4ebbc3bb335eb93f63643b71d9ffce26155a64ac36d0179acb7dc71a0c94415` | This test case succeeded with other compilation modes: {'Y-'} | -| `simple/yul_instructions/blockhash.sol::4::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:255")]) but got 0x52d8ee9edfe1a898555373042193e5b957f4b67f4c1c2bc5038bbc0c7253668d` | This test case succeeded with other compilation modes: {'E-', 'E+'} | -| `simple/yul_instructions/blockhash.sol::4::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$BLOCK_HASH:255")]) but got 0xf84672ca120abb8236e22bf8307ac96051277c332355cadfd08b5852b598a71a` | This test case succeeded with other compilation modes: {'E-', 'E+'} | -| `simple/yul_instructions/difficulty.sol::0::Y- =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/difficulty.sol::0::Y+ =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/difficulty.sol::0::E+ =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/difficulty.sol::0::E- =0.8.17` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/prevrandao.sol::0::Y-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/prevrandao.sol::0::Y+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/prevrandao.sol::0::E+` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `simple/yul_instructions/prevrandao.sol::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Compound([CalldataItem("$DIFFICULTY")]) but got 0x0000000000000000000000000000000000000000000000000008e1bc9bf04000` | | -| `translated_semantic_tests/array/array_storage_index_access/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/array/array_storage_length_access/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/array/array_storage_push_empty_length_address/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/array/array_storage_push_pop/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/array/copying/copy_removes_bytes_data/test.json::0::E-` | `Failed to execute all of the steps on the driver: Storage Empty Assertion Step Failed: server returned an error response: error code -32601: Method not found` | | -| `translated_semantic_tests/array/slices/array_slice_calldata_to_memory/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle the function call execution: Transaction inclusion watching timeout for 0xda5d8902f33d434b0e305acbed970dbadb0e1595cc7c3d1f097268ab95cab4aa: transaction was not confirmed within the timeout` | | -| `translated_semantic_tests/functionCall/call_options_overload/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Calldata assertion failed - Expected Single(0x0000000000000000000000000000000000000000000000000de0b6b3a7640000) but got 0x0000000000000000000000000000000000000000000000000de0b6b3a754bdde` | | -| `translated_semantic_tests/functionCall/gas_and_value_basic/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected false but got true. Revert reason: None` | | -| `translated_semantic_tests/inheritance/inherited_function/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to deploy contracts for the function call step: Failed to get or deploy contract instance during input execution: Failed to deploy contract: Transaction inclusion watching timeout for 0x0d02f35e79908efcd422ce577a710f025fd31d0cab67ecc62f6ca3bb6fef0baf: transaction was not confirmed within the timeout` | | -| `translated_semantic_tests/inheritance/overloaded_function_call_with_if_else/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to deploy contracts for the function call step: Failed to get or deploy contract instance during input execution: Failed to deploy contract: Transaction inclusion watching timeout for 0x1aad59e22cffaafb040e8d96b7723f7cb0a7d09c9164218785831b4dba4ad74e: transaction was not confirmed within the timeout` | | -| `translated_semantic_tests/inlineAssembly/blobhash_index_exceeding_blob_count/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/libraries/external_call_with_storage_mapping_parameter/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/state/block_blobbasefee/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: None` | | -| `translated_semantic_tests/various/selfdestruct_post_cancun/test.json::0::E-` | `Failed to execute all of the steps on the driver: Function call step Failed: Failed to handle function call assertions: Transaction status assertion failed - Expected true but got false. Revert reason: Some("panic: assertion failed (0x01)")` | | -| `translated_semantic_tests/various/selfdestruct_post_cancun_multiple_beneficiaries/test.json::0::E-` | `Failed to execute all of the steps on the driver: Balance Assertion Step Failed: Balance assertion failed - Expected 1000000000000000000 but got 0 for 0x1111111111111111111111111111111111111111 resolved to 0x1111111111111111111111111111111111111111` | | - - -
From 72b3b92fac208a1c5409ae9898abf41ac8699997 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Sun, 2 Nov 2025 20:58:09 +0300 Subject: [PATCH 22/24] Update retester to supply the failure reason --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 2fcf956e3cdd3..c73152fa1efb3 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: 33b5faca4529b659f427b80080bcae95bbb1bfd6 + ref: 2af1a62319334ce91fa9f8597e19b8360adb53a9 path: revive-differential-tests submodules: recursive - name: Installing Retester From 3d6d1fc4c172dcf4a0fe587eab22c564e93b9752 Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Mon, 3 Nov 2025 00:30:15 +0300 Subject: [PATCH 23/24] Update the version of dt --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index c73152fa1efb3..2b0458f3ee97e 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: paritytech/revive-differential-tests - ref: 2af1a62319334ce91fa9f8597e19b8360adb53a9 + ref: 75159229dff6e3cc0b1837c95fb053f7bb2a7cb5 path: revive-differential-tests submodules: recursive - name: Installing Retester From c149a67d3917cefbf9f0a6835f0226689f8fa80b Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Mon, 3 Nov 2025 18:16:29 +0300 Subject: [PATCH 24/24] Chane the repo used for caches --- .github/workflows/tests-evm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-evm.yml b/.github/workflows/tests-evm.yml index 2b0458f3ee97e..b193e060bfe56 100644 --- a/.github/workflows/tests-evm.yml +++ b/.github/workflows/tests-evm.yml @@ -47,7 +47,7 @@ jobs: run: mkdir workdir - name: Downloading & Initializing the compilation caches run: | - curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/0xOmarA/revive-differential-tests-compiler-cache/releases/download/v1.0.0/cache.tar.gz" + curl -fL --retry 3 --retry-all-errors --connect-timeout 10 -o cache.tar.gz "https://github.com/paritytech/revive-differential-tests/releases/download/compilation-caches-v1.0/cache.tar.gz" tar -zxf cache.tar.gz -C ./workdir - name: Running the Differential Tests run: |