Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions .github/scripts/check-zombienet-flaky-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ line_num=0

while IFS= read -r line || [[ -n "$line" ]]; do
line_num=$((line_num + 1))

if [[ -z "$line" ]]; then
continue
fi

# Parse format: test-name:issue-number
if [[ ! "$line" =~ ^([^:]+):([0-9]+)$ ]]; then
echo "❌ Line $line_num: Missing required issue number" >&2
Expand All @@ -42,26 +42,26 @@ while IFS= read -r line || [[ -n "$line" ]]; do
has_errors=true
continue
fi

test_name="${BASH_REMATCH[1]}"
issue_number="${BASH_REMATCH[2]}"

set +e
issue_data=$(gh issue view "$issue_number" --json state,title,url 2>&1)
gh_exit_code=$?
set -e

if [[ $gh_exit_code -ne 0 ]]; then
echo "❌ Line $line_num: Issue #$issue_number does not exist" >&2
echo " Test: $test_name" >&2
has_errors=true
continue
fi

url=$(echo "$issue_data" | jq -r '.url')
state=$(echo "$issue_data" | jq -r '.state')
title=$(echo "$issue_data" | jq -r '.title')

# Check if it's an issue (not a PR) by verifying the URL contains '/issues/'
if [[ ! "$url" =~ /issues/ ]]; then
echo "❌ Line $line_num: #$issue_number is a Pull Request, not an Issue" >&2
Expand All @@ -71,16 +71,17 @@ while IFS= read -r line || [[ -n "$line" ]]; do
has_errors=true
continue
fi

if [[ "$state" == "OPEN" ]]; then
echo "✅ Line $line_num: $test_name -> Issue #$issue_number (open)"
else
echo "⚠️ Line $line_num: Issue #$issue_number is closed: '$title'" >&2
echo " Test: $test_name" >&2
echo " Consider removing this entry if the issue is resolved." >&2
fi

done < "$FLAKY_TESTS_FILE"

# exclude empty lines and comments
done < <(grep -vE '^\s*(#|$)' "$FLAKY_TESTS_FILE")

echo

Expand Down
7 changes: 7 additions & 0 deletions .github/zombienet-flaky-tests
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,10 @@ zombienet-substrate-0002-validators-warp-sync:8871
zombienet-cumulus-0009-elastic_scaling_pov_recovery:8986
zombienet-cumulus-0010-elastic_scaling_multiple_block_per_slot:8999
zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility:9980
# disabled until zombienet support penpal (https://github.com/paritytech/zombienet-sdk/issues/480)
zombienet-polkadot-shared-core-idle-parachain:10653
zombienet-polkadot-functional-async-backing-6-seconds-rate:10653
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We probably can remove this at all, because cumulus tests cover this.

zombienet-polkadot-elastic-scaling-doesnt-break-parachains:10653
zombienet-polkadot-smoke-0005-precompile-pvf-smoke:10653
zombienet-polkadot-smoke-0004-coretime-smoke-test:10653
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will need to use westend-coretime because it tests coretime.

zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke:10653
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We also test runtime upgrade in multiple cumulus tests, aka should not be required here.

Loading