chore: Report flakes in CI

Author: cdata

.github/workflows/report_test_flakes.yaml

@@ -0,0 +1,86 @@
+on:
+  workflow_call:
+    inputs:
+      target:
+        required: true
+        type: string
+    secrets:
+      GITHUB_TOKEN:
+        required: true
+
+name: 'Workflow Analysis'
+
+jobs:
+  report-test-flakes:
+    name: 'Report test flakes'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Parse test results
+        run: |
+          RESULTS_PATH="./test-results-${{ inputs.target }}/test-results.log"
+
+          csplit -q "$RESULTS_PATH" %^------------%
+
+          if [[ -f "./xx00" ]]; then
+            SUMMARY=`tail ./xx00 -n+2`
+
+            if [[ "$SUMMARY" =~ 'FLAKY' ]]; then
+              # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
+              {
+                echo "summary<<EOF"
+                echo "$SUMMARY"
+                echo "EOF"
+              } >> $GITHUB_OUTPUT
+            fi
+          fi
+      - name: Report flakes
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const { data: comments } = await github.rest.issues.listComments({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+          });
+
+          const flakeyTestsHeader = 'Test flake analysis: ${{ inputs.target }}';
+          const existingComment = comments.find(comment => {
+            return comment.user.type === 'Bot' && comment.body.includes(flakeyTestsHeader)
+          });
+
+          let body;
+
+          if (${{ steps['Parse test results'].outputs.summary }}) {
+            body = `${flakeyTestsHeader}
+
+            \`\`\`shell
+            ${{ steps['Parse test results'].outputs.summary }}
+            \`\`\`
+            `;
+          } else {
+            body = `${flakeyTestsHeader}
+
+            No flakes detected!
+            `;
+          }
+
+          if (existingComment) {
+            github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: existingComment.id,
+              body
+            })
+          } else {
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            })
+          }

.github/workflows/run_test_suite.yaml

@@ -79,7 +79,7 @@ jobs:
           ipfs_version: v0.17.0
           run_daemon: true
       - name: 'Run Rust native target tests'
-        run: cargo nextest run --features test-kubo,headers --retries 5 --color always
+        run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log
         env:
           NOOSPHERE_LOG: academic
 
@@ -106,9 +106,21 @@ jobs:
           ipfs_version: v0.17.0
           run_daemon: true
       - name: 'Run Rust native target tests'
-        run: cargo nextest run --features test-kubo,headers --retries 5 --color always
+        run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log
         env:
           NOOSPHERE_LOG: academic
+      - uses: actions/upload-artifact@v3
+        with:
+          name: test-results-linux
+          path: ./test-results.log
+
+  report-test-flakes:
+    name: 'Report test flakes (Linux)'
+    needs: ['run-test-suite-linux']
+    uses: ./.github/workflows/report_test_flakes.yaml
+    secrets: inherit
+    with:
+      target: 'linux'
 
   run-test-suite-linux-rocksdb:
     runs-on: ubuntu-latest