Skip to content

run system test from private repo #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bvolovat merged 1 commit into from
Jan 29, 2026
Merged

run system test from private repo #700

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 153 additions & 56 deletions .github/workflows/incluster-comp-pr-merged.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,76 +163,173 @@ jobs:
input: ${{ inputs.REQUIRED_TESTS }}

run-tests:
strategy:
fail-fast: false
matrix:
TEST: ${{ fromJson(needs.docker-build.outputs.TEST_NAMES) }}
needs: docker-build
if: ${{ needs.docker-build.outputs.TEST_NAMES != '' && toJson(fromJson(needs.docker-build.outputs.TEST_NAMES)) != '[]' }}
if: ${{ inputs.HELM_E2E_TEST == true }}
runs-on: ubuntu-latest
steps:
- name: Checkout systests repo
uses: actions/checkout@v4
with:
repository: armosec/system-tests
ref: ${{ inputs.SYSTEM_TESTS_BRANCH }}
path: .
- name: Set dispatch info
id: dispatch-info
run: |
# Correlation ID WITHOUT attempt - so re-runs can find the original run
CORRELATION_ID="${GITHUB_REPOSITORY##*/}-${{ github.run_id }}"
echo "correlation_id=${CORRELATION_ID}" >> "$GITHUB_OUTPUT"
echo "Correlation ID: ${CORRELATION_ID}, Attempt: ${{ github.run_attempt }}"

- uses: actions/setup-python@v5
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
python-version: "3.9"
cache: "pip"
app-id: ${{ secrets.E2E_DISPATCH_APP_ID }}
private-key: ${{ secrets.E2E_DISPATCH_APP_PRIVATE_KEY }}
owner: armosec
repositories: shared-workflows

- name: create env
run: ./create_env.sh

- name: Generate uuid
id: uuid
- name: Dispatch Helm E2E to private repo
if: ${{ github.run_attempt == 1 }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
REQUIRED_TESTS_JSON: ${{ inputs.REQUIRED_TESTS }}
run: |
echo "RANDOM_UUID=systets-$(uuidgen)" >> $GITHUB_OUTPUT
gh api "repos/armosec/shared-workflows/dispatches" \
-f event_type="e2e-test-trigger" \
-f "client_payload[correlation_id]=${{ steps.dispatch-info.outputs.correlation_id }}" \
-f "client_payload[github_repository]=${GITHUB_REPOSITORY}" \
-f "client_payload[environment]=production" \
-f "client_payload[tests]=${REQUIRED_TESTS_JSON}" \
-f "client_payload[systests_branch]=${{ inputs.SYSTEM_TESTS_BRANCH }}" \
-f "client_payload[in_cluster_chart_branch]=${{ inputs.HELM_BRANCH }}" \
-f "client_payload[ks_branch]=release" \
-f "client_payload[charts_name]=kubescape-operator" \
-f "client_payload[charts_repo]=${GITHUB_REPOSITORY}"

- name: Create k8s Kind Cluster
id: kind-cluster-install
uses: helm/kind-action@v1
with:
cluster_name: ${{ steps.uuid.outputs.RANDOM_UUID }}
# kubectl_version: v1.23.12

- name: run-tests
- name: Find E2E workflow run
id: find-run
env:
CUSTOMER: ${{ secrets.CUSTOMER }}
USERNAME: ${{ secrets.USERNAME }}
PASSWORD: ${{ secrets.PASSWORD }}
CLIENT_ID: ${{ secrets.CLIENT_ID_PROD }}
SECRET_KEY: ${{ secrets.SECRET_KEY_PROD }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
QUAY_REGISTRY_ACCESS_TOKEN: "${{ secrets.QUAY_REGISTRY_ACCESS_TOKEN }}"
AZURE_REGISTRY_ACCESS_TOKEN: "${{ secrets.AZURE_REGISTRY_ACCESS_TOKEN }}"
AWS_REGISTRY_SECRET_KEY: "${{ secrets.AWS_REGISTRY_SECRET_KEY }}"
GOOGLE_REGISTRY_KEY: "${{ secrets.GOOGLE_REGISTRY_KEY }}"
GH_TOKEN: ${{ steps.app-token.outputs.token }}
CORRELATION_ID: ${{ steps.dispatch-info.outputs.correlation_id }}
run: |
echo "Test history:"
echo " ${{ matrix.TEST }} " >/tmp/testhistory
cat /tmp/testhistory
source systests_python_env/bin/activate
for i in {1..15}; do
run_id=$(gh api "repos/armosec/shared-workflows/actions/runs?event=repository_dispatch&per_page=30" \
--jq '.workflow_runs | map(select(.name | contains("'"$CORRELATION_ID"'"))) | first | .id // empty')

if [ -n "$run_id" ]; then
echo "run_id=${run_id}" >> "$GITHUB_OUTPUT"
gh api "repos/armosec/shared-workflows/actions/runs/${run_id}" --jq '"url=" + .html_url' >> "$GITHUB_OUTPUT"
exit 0
fi
echo "Attempt $i: waiting for run..."
sleep $((i < 5 ? 10 : 30))
done
echo "::error::Could not find workflow run"
exit 1

python3 systest-cli.py \
-t ${{ matrix.TEST }} \
-b production \
-c CyberArmorTests \
--logger DEBUG \
--kwargs helm_branch=${{ inputs.HELM_BRANCH }} \
${{ inputs.COMPONENT_NAME }}-tag=${{ needs.docker-build.outputs.IMAGE_TAG_PRERELEASE }}
- name: Re-run failed jobs in private repo
id: rerun
if: ${{ github.run_attempt > 1 }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
RUN_ID: ${{ steps.find-run.outputs.run_id }}
run: |
conclusion=$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.conclusion')
echo "Previous conclusion: $conclusion"

if [ "$conclusion" = "success" ]; then
echo "Previous run passed. Nothing to re-run."
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi

# Full rerun if cancelled, partial if failed
if [ "$conclusion" = "cancelled" ]; then
echo "Run was cancelled - triggering full re-run"
gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun"
else
echo "Re-running failed jobs only"
gh api --method POST "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/rerun-failed-jobs"
fi

# Wait for status to flip from 'completed'
for i in {1..30}; do
[ "$(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" --jq '.status')" != "completed" ] && break
sleep 2
done

deactivate
- name: Wait for E2E tests to complete
if: ${{ steps.rerun.outputs.skip != 'true' }}
env:
GH_TOKEN: ${{ steps.app-token.outputs.token }}
RUN_ID: ${{ steps.find-run.outputs.run_id }}
URL: ${{ steps.find-run.outputs.url }}
run: |
echo "Monitoring: ${URL}"

for i in {1..60}; do # 60 iterations × 60s = 1 hour max
read status conclusion < <(gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}" \
--jq '[.status, .conclusion // "null"] | @tsv')

echo "Status: ${status} | Conclusion: ${conclusion}"

if [ "$status" = "completed" ]; then
if [ "$conclusion" = "success" ]; then
echo "E2E tests passed!"
exit 0
fi

echo "::error::E2E tests failed: ${conclusion}"
echo ""

# Get failed job IDs to a file first
gh api "repos/armosec/shared-workflows/actions/runs/${RUN_ID}/jobs" \
--jq '.jobs[] | select(.conclusion == "failure") | [.id, .name, (.steps[] | select(.conclusion == "failure") | .name)] | @tsv' > /tmp/failed_jobs.txt

# Process each failed job
while IFS=$'\t' read -r job_id job_name step_name; do
# Extract test name: "run-helm-e2e / ST (relevancy_python)" → "relevancy_python"
test_name=$(echo "$job_name" | sed 's/.*(\(.*\))/\1/')

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "${job_name}"
echo " Step: ${step_name}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Fetch logs to temp file
gh api "repos/armosec/shared-workflows/actions/jobs/${job_id}/logs" 2>/dev/null > /tmp/job_logs.txt

# Show summary in console
grep -E "(ERROR|FAILURE)" /tmp/job_logs.txt | tail -10
echo ""

# Save to separate file per test
log_file="failed_${test_name}.txt"
echo "════════════════════════════════════════" > "$log_file"
echo "${job_name}" >> "$log_file"
echo " Step: ${step_name}" >> "$log_file"
echo "════════════════════════════════════════" >> "$log_file"
last_endgroup=$(grep -n "##\[endgroup\]" /tmp/job_logs.txt | tail -1 | cut -d: -f1)
if [ -n "$last_endgroup" ]; then
tail -n +$((last_endgroup + 1)) /tmp/job_logs.txt >> "$log_file"
else
tail -500 /tmp/job_logs.txt >> "$log_file"
fi
done < /tmp/failed_jobs.txt

echo "View full logs: ${URL}"
exit 1
fi

sleep 60
done

echo "::error::Timeout waiting for tests"
exit 1

- name: Test Report
uses: mikepenz/action-junit-report@v5
if: always() # always run even if the previous step fails
- name: Upload failed step logs
if: failure()
uses: actions/upload-artifact@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
report_paths: "**/results_xml_format/**.xml"
name: failed-e2e-logs-attempt-${{ github.run_attempt }}
path: failed_*.txt
retention-days: 7

create-release-and-retag:
if: ${{ contains(github.event.pull_request.labels.*.name, 'release') && always() && contains(needs.*.result, 'success') && !(contains(needs.*.result, 'failure')) && !(contains (needs.*.result,'cancelled')) || inputs.FORCE }}
Expand Down