ci: add Multus node-density-cni e2e with PR performance reports

.github/workflows/kind-e2e.yml

@@ -104,6 +104,20 @@ jobs:
         working-directory: ./e2e
         run: ./test-subdirectory-chaining-passthru.sh
 
+      - name: Test kube-burner node-density-cni
+        if: ${{ matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        working-directory: ./e2e
+        run: CNI_VERSION=${{ matrix.cni-version }} ./test-node-density-cni.sh
+
+      - name: Upload node-density-cni performance test data
+        if: ${{ always() && matrix.multus-manifest == 'multus-daemonset-thick.yml' }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          # Consumed by the performance-report workflow (ovn-k style).
+          name: node-density-cni-${{ matrix.cni-version }}-performance-test-data-${{ github.run_id }}
+          path: e2e/perf-data/
+          if-no-files-found: warn
+
       - name: Export kind logs
         if: always()
         run: |

.github/workflows/performance-report.yml

@@ -0,0 +1,222 @@
+name: performance-report
+
+on:
+  workflow_run:
+    workflows: ["e2e-kind"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+  actions: read
+
+concurrency:
+  group: perf-report-${{ github.event.workflow_run.head_branch || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  process-performance-results:
+    name: Process Performance Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure'
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r e2e/perf/requirements.txt
+
+      - name: Download performance test data artifacts
+        run: |
+          e2e/perf/download-artifacts.py \
+            --run-id ${{ github.event.workflow_run.id }} \
+            --filter "performance-test-data" \
+            --output-dir artifact-downloads \
+            --extract
+
+      - name: Organize extracted artifacts and generate JSON data
+        run: |
+          for dir in artifact-downloads/*performance-test-data*; do
+            if [ -d "$dir" ]; then
+              workload=$(basename "$dir" | sed 's/-performance-test-data-.*//')
+              mkdir -p "reports/${workload}"
+
+              metrics_dir=""
+              if [ -d "$dir/perf-data/metrics" ]; then
+                metrics_dir="$dir/perf-data/metrics"
+              elif [ -d "$dir/metrics" ]; then
+                metrics_dir="$dir/metrics"
+              fi
+
+              if [ -n "$metrics_dir" ]; then
+                echo "Processing workload: $workload (metrics: $metrics_dir)"
+                python3 e2e/perf/generate_perf_report.py \
+                  --workload "$workload" \
+                  --metrics-dir "$metrics_dir" \
+                  --json-output "reports/${workload}/performance_data.json" \
+                  --output "reports/${workload}/performance_report.md"
+              else
+                echo "Warning: no metrics directory found under $dir"
+                find "$dir" -type f | head -50 || true
+              fi
+            fi
+          done
+
+          echo "=== Generated reports structure ==="
+          find reports -type f 2>/dev/null || true
+
+      - name: Get associated PRs
+        id: get_prs
+        run: |
+          PR_NUMBERS=$(e2e/perf/get-pr-info.py \
+            --event-path $GITHUB_EVENT_PATH \
+            --format json)
+          echo "pr_numbers=$PR_NUMBERS" >> $GITHUB_OUTPUT
+          echo "Found PRs: $PR_NUMBERS"
+
+      - name: Fetch baseline run
+        id: get_baseline
+        if: steps.get_prs.outputs.pr_numbers != '[]'
+        run: |
+          # Prefer a successful push run of e2e-kind as the baseline when no
+          # scheduled performance lane exists yet.
+          BASELINE_INFO=$(e2e/perf/get-baseline-run.py \
+            --workflow kind-e2e.yml \
+            --event push \
+            --output json) || echo "{}"
+
+          BASELINE_ID=$(echo "$BASELINE_INFO" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('id', ''))" 2>/dev/null || echo "")
+          BASELINE_URL=$(echo "$BASELINE_INFO" | python3 -c "import sys, json; data=json.load(sys.stdin); print(data.get('url', ''))" 2>/dev/null || echo "")
+
+          echo "baseline_run_id=$BASELINE_ID" >> $GITHUB_OUTPUT
+          echo "baseline_run_url=$BASELINE_URL" >> $GITHUB_OUTPUT
+
+          echo "Baseline run ID: $BASELINE_ID"
+          echo "Baseline run URL: $BASELINE_URL"
+
+      - name: Download baseline artifacts
+        if: steps.get_baseline.outputs.baseline_run_id != ''
+        run: |
+          e2e/perf/download-artifacts.py \
+            --run-id ${{ steps.get_baseline.outputs.baseline_run_id }} \
+            --filter "performance-test-data" \
+            --output-dir baseline-downloads \
+            --extract
+
+      - name: Organize baseline artifacts and generate JSON data
+        if: steps.get_baseline.outputs.baseline_run_id != ''
+        run: |
+          mkdir -p baseline-reports
+          for dir in baseline-downloads/*performance-test-data*; do
+            if [ -d "$dir" ]; then
+              workload=$(basename "$dir" | sed 's/-performance-test-data-.*//')
+              mkdir -p "baseline-reports/${workload}"
+
+              metrics_dir=""
+              if [ -d "$dir/perf-data/metrics" ]; then
+                metrics_dir="$dir/perf-data/metrics"
+              elif [ -d "$dir/metrics" ]; then
+                metrics_dir="$dir/metrics"
+              fi
+
+              if [ -n "$metrics_dir" ]; then
+                echo "Processing baseline workload: $workload"
+                python3 e2e/perf/generate_perf_report.py \
+                  --workload "$workload" \
+                  --metrics-dir "$metrics_dir" \
+                  --json-output "baseline-reports/${workload}/performance_data.json" \
+                  --output "baseline-reports/${workload}/performance_report.md"
+              fi
+            fi
+          done
+
+          echo "=== Baseline reports structure ==="
+          find baseline-reports -type f 2>/dev/null || echo "No baseline reports found"
+
+      - name: Compare reports with baseline
+        if: steps.get_prs.outputs.pr_numbers != '[]'
+        run: |
+          mkdir -p enhanced-reports
+
+          for workload_dir in reports/*/; do
+            if [ -d "$workload_dir" ]; then
+              workload=$(basename "$workload_dir")
+              current_json="$workload_dir/performance_data.json"
+              baseline_json="baseline-reports/$workload/performance_data.json"
+              enhanced_report="enhanced-reports/$workload/performance_report.md"
+
+              if [ -f "$current_json" ]; then
+                echo "Comparing data for workload: $workload"
+                mkdir -p "enhanced-reports/$workload"
+
+                if [ -f "$baseline_json" ]; then
+                  e2e/perf/compare-reports.py \
+                    --current "$current_json" \
+                    --baseline "$baseline_json" \
+                    --baseline-url "${{ steps.get_baseline.outputs.baseline_run_url }}" \
+                    --output "$enhanced_report"
+                else
+                  echo "No baseline found for $workload, using current data as-is"
+                  e2e/perf/compare-reports.py \
+                    --current "$current_json" \
+                    --output "$enhanced_report"
+                fi
+              else
+                echo "Warning: No JSON data found for $workload"
+              fi
+            fi
+          done
+
+      - name: Post performance report to PRs
+        if: steps.get_prs.outputs.pr_numbers != '[]'
+        run: |
+          PR_NUMBERS=$(echo '${{ steps.get_prs.outputs.pr_numbers }}' | python3 -c "import sys, json; print(' '.join(f'--pr {pr}' for pr in json.load(sys.stdin)))")
+
+          if [ -n "$PR_NUMBERS" ]; then
+            e2e/perf/post-pr-comment.py \
+              $PR_NUMBERS \
+              --reports-dir enhanced-reports \
+              --run-id "${{ github.event.workflow_run.id }}" \
+              --run-url "${{ github.event.workflow_run.html_url }}" \
+              --status "${{ github.event.workflow_run.conclusion }}" \
+              --baseline-id "${{ steps.get_baseline.outputs.baseline_run_id }}" \
+              --baseline-url "${{ steps.get_baseline.outputs.baseline_run_url }}"
+          fi
+
+      - name: Upload processed artifacts
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: processed-performance-data-${{ github.event.workflow_run.id }}
+          path: |
+            reports/
+            enhanced-reports/
+            baseline-reports/
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## Performance Report Processing Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- **Workflow Run ID:** ${{ github.event.workflow_run.id }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Conclusion:** ${{ github.event.workflow_run.conclusion }}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Associated PRs:** ${{ steps.get_prs.outputs.pr_numbers }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Artifacts Processed" >> $GITHUB_STEP_SUMMARY
+          PERF_TEST_COUNT=$(ls -1d artifact-downloads/*performance-test-data* 2>/dev/null | wc -l | tr -d ' ')
+          echo "- Performance test data dirs: **${PERF_TEST_COUNT}**" >> $GITHUB_STEP_SUMMARY
+          find reports enhanced-reports -type f 2>/dev/null | sed 's/^/- /' >> $GITHUB_STEP_SUMMARY || true

.gitignore

@@ -3,6 +3,8 @@ bin/
 e2e/bin/
 e2e/yamls/
 e2e/repos/
+e2e/perf-data/
+e2e/node-density-cni-metrics.tgz
 
 # GOPATH created by the build script
 gopath/

e2e/README.md

@@ -35,6 +35,32 @@ $ ./setup_cluster.sh
 $ ./test-simple-macvlan1.sh
 ```
 
+### Kube-burner node-density-cni (Multus)
+
+After `setup_cluster.sh` with the thick Multus daemonset, you can run a CI-sized
+Multus-aware node-density-cni workload (secondary macvlan on `eth1`):
+
+```
+$ ./test-node-density-cni.sh
+```
+
+Override scale with `PODS_PER_NODE` (default `64`).
+
+The run enables kube-burner `podLatency` and writes metrics under
+`perf-data/metrics/` (including `podLatencyMeasurement-node-density-cni-<cniVersion>.json`)
+for the OVN-K-style [`performance-report`](../.github/workflows/performance-report.yml)
+workflow. In GitHub Actions this density lane runs for all CNI versions on the
+thick Multus matrix cells; on completion, `performance-report` posts a Pod Ready
+Latency summary as a PR comment.
+
+Pass `CNI_VERSION` to match the cluster CNI config (default `0.4.0`):
+
+```
+$ CNI_VERSION=0.3.1 ./test-node-density-cni.sh
+```
+
+See also [`e2e/perf/README.md`](perf/README.md).
+
 ### How to teardown cluster
 
 ```

e2e/get_tools.sh

@@ -14,3 +14,10 @@ chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq
 wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm
+
+KUBE_BURNER_VERSION=V2.7.3
+curl -L --fail -o /tmp/kube-burner.tar.gz \
+	"https://github.com/kube-burner/kube-burner/releases/download/v2.7.3/kube-burner-${KUBE_BURNER_VERSION}-linux-x86_64.tar.gz"
+tar -xz -C ./bin -f /tmp/kube-burner.tar.gz kube-burner
+rm -f /tmp/kube-burner.tar.gz
+chmod +x ./bin/kube-burner

e2e/kubeburner/node-density-cni.yml

@@ -0,0 +1,51 @@
+---
+# Multus-aware adaptation of kube-burner kubelet-density-cni.
+# Each iteration creates a webserver Deployment, Service, and curl client
+# Deployment. Pods are annotated for a Multus secondary macvlan network.
+metricsEndpoints:
+  - alias: local-indexer
+    indexer:
+      type: local
+      metricsDirectory: perf-data/metrics
+      createTarball: true
+      tarballName: node-density-cni-metrics.tgz
+
+global:
+  gc: true
+  timeout: 30m
+  measurements:
+    - name: podLatency
+
+jobs:
+  - name: create-density-nad
+    jobIterations: 1
+    qps: 5
+    burst: 5
+    namespacedIterations: false
+    namespace: node-density-cni
+    waitWhenFinished: true
+    cleanup: true
+    objects:
+      - objectTemplate: kubeburner/templates/density-macvlan-nad.yml
+        replicas: 1
+
+  - name: node-density-cni
+    # Overridden by test-node-density-cni.sh via --set jobs.1.jobIterations
+    jobIterations: 64
+    qps: 10
+    burst: 10
+    namespacedIterations: false
+    namespace: node-density-cni
+    waitWhenFinished: true
+    podWait: false
+    cleanup: false
+    preLoadImages: true
+    preLoadPeriod: 2m
+    maxWaitTimeout: 15m
+    objects:
+      - objectTemplate: kubeburner/templates/webserver-deployment.yml
+        replicas: 1
+      - objectTemplate: kubeburner/templates/webserver-service.yml
+        replicas: 1
+      - objectTemplate: kubeburner/templates/curl-deployment.yml
+        replicas: 1

e2e/kubeburner/templates/curl-deployment.yml

@@ -0,0 +1,42 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: curl-{{.Replica}}-{{.Iteration}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: curl-{{.Replica}}-{{.Iteration}}
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        name: curl-{{.Replica}}-{{.Iteration}}
+      annotations:
+        k8s.v1.cni.cncf.io/networks: density-macvlan
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/worker: ""
+      containers:
+        - name: curlapp
+          image: quay.io/cloud-bulldozer/curl:latest
+          command: ["sleep", "inf"]
+          env:
+            - name: WEBSERVER_HOSTNAME
+              value: webserver-{{.Replica}}-{{.Iteration}}
+            - name: WEBSERVER_PORT
+              value: "8080"
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            privileged: false
+          startupProbe:
+            exec:
+              command:
+                - "/bin/sh"
+                - "-c"
+                - "curl --fail --silent --show-error ${WEBSERVER_HOSTNAME}:${WEBSERVER_PORT}"
+            periodSeconds: 1
+            timeoutSeconds: 1
+            failureThreshold: 600
+      restartPolicy: Always