Dynamically generate compatibility test files #7547
Conversation
…n test to workflow
chatton
requested review from
AdityaSripal,
damiannolan,
DimitrisJim,
gjermundgaraba and
bznein
as code owners
| @@ -111,6 +112,7 @@ func (s *InterchainAccountsParamsTestSuite) TestControllerEnabledParam() { | |||
| }) | |||
| } | |||
|
|
|||
| // compatibility:TestHostEnabledParam:from_versions: v9.0.0,v8.4.0,v7.5.0 | |||
There was a problem hiding this comment.
this syntax is arbitrary, we can change this to whatever we like. I figured it is a straightforward enough way of specifying custom data about specific tests.
| if "(t *testing.T)" in line: | ||
| return re.search(r"func\s+(.*)\(", line).group(1) |
There was a problem hiding this comment.
a little bit gross, we can try and come up with a cleaner way of doing this if people feel strongly about it.
| version = version[1:] | ||
| if version.startswith("release-"): | ||
| # strip off the release prefix and parse the actual version | ||
| return parse_version(version[len("release-"):]) |
There was a problem hiding this comment.
[nit] for consistency, we can either do above return parse_version(version[1:]) or here version = version[len("release-"):]
| if version.startswith("v"): | ||
| # semver versions do not include a "v" prefix. | ||
| version = version[1:] | ||
| if version.startswith("release-"): |
There was a problem hiding this comment.
[nit] maybe extract "release" to const?
| entry = (release_version, version, test, entrypoint, relayer, chain_image) | ||
|
|
||
| if entry not in seen and chain in ("chain-a", "all"): | ||
| include_entries.append( | ||
| { | ||
| "chain-a": release_version, | ||
| "chain-b": version, | ||
| "entrypoint": entrypoint, | ||
| "test": test, | ||
| "relayer-type": relayer, | ||
| "chain-image": chain_image | ||
| } | ||
| ) | ||
| seen.add(entry) | ||
|
|
||
| entry = (version, release_version, test, entrypoint, relayer, chain_image) | ||
| if entry not in seen and chain in ("chain-b", "all"): | ||
| include_entries.append( | ||
| { | ||
| "chain-a": version, | ||
| "chain-b": release_version, | ||
| "entrypoint": entrypoint, | ||
| "test": test, | ||
| "relayer-type": relayer, | ||
| "chain-image": chain_image | ||
| } | ||
| ) | ||
| seen.add(entry) |
There was a problem hiding this comment.
[q] could we refactor this so that there is less duplication?
| raise ValueError(f"key {k} not found in {item.keys()}") | ||
|
|
||
| if len(compatibility_json["include"]) > 256: | ||
| # if this error occurs, split out the workflow into two jobs, one for chain-a and one for chain-b |
There was a problem hiding this comment.
maybe we can add this info to the error itself so that we don't have to go look at the code?
|
Description
closes: #4872
This PR does the following
Here is a manual trigger I ran last night https://github.com/cosmos/ibc-go/actions/runs/11806430988/job/32912856842
I've since addressed the failing tests, but won't run again until tonight to not interfere with everyone else since it spins up a huge number of runners. The issues fixed were
I think we can still merge this PR if people are happy with the general approach, and then I can do a follow up to fix any issues that still persist after running the tests again tonight.
I dedicate this PR to @crodriguezvega who dealt with these compatibility files for far too long! 🫡
Before we can merge this PR, please make sure that all the following items have been
checked off. If any of the checklist items are not applicable, please leave them but
write a little note why.
docs/).godoccomments.Files changedin the GitHub PR explorer.SonarCloud Reportin the comment section below once CI passes.