ci: retry flaky tests

.github/workflows/ci.yml

@@ -81,13 +81,13 @@ jobs:
         run: pnpm run build
 
       - name: Test unit
-        run: pnpm run test-unit
+        run: bash ./scripts/retry.sh "pnpm --color=always test-unit"
 
       - name: Test serve
-        run: pnpm run test-serve
+        run: bash ./scripts/retry.sh "pnpm --color=always test-serve"
 
       - name: Test build
-        run: pnpm run test-build
+        run: bash ./scripts/retry.sh "pnpm --color=always test-build"
 
       - name: Test docs
         run: pnpm run test-docs

scripts/retry.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#######################################################################
+# Runs command, retrying if specific error is logged (see ERRORS below)
+#
+# Usage: retry.sh <cmd> [<retries>]
+#
+# Arguments:
+#  cmd        command to run
+#  retries    number of times to retry command (default: 3)
+#######################################################################
+COMMAND_TO_RUN=${1?Missing command argument}
+NUM_RETRIES=${2:-3}
+
+FG_BOLD_WHITE='\033[1;37m'
+FG_RED='\033[0;31m'
+FG_BLUE='\033[0;34m'
+FG_GRAY='\033[1;30m'
+BG_RED='\033[41m'
+BG_BLUE='\033[44m'
+NC='\033[0m' # No Color
+
+function findErrors() {
+  FILE_TO_CHECK=${1?Missing file argument}
+
+  # Node errors seen in Vitest (vitejs/vite#9492)
+  ERRORS=(
+    "Check failed: result.second."                        # nodejs/node#43617
+    "FATAL ERROR: v8::FromJust Maybe value is Nothing."   # vitest-dev/vitest#1191
+  )
+
+  for error in "${ERRORS[@]}"; do
+    if grep -qnr "${error}" ${FILE_TO_CHECK}; then
+      echo ${error}
+    fi
+  done
+}
+
+for i in `seq ${NUM_RETRIES}`; do
+  ${COMMAND_TO_RUN} 2>&1 | tee ${NPM_SCRIPT_COMMAND}.log
+
+  error=$(findErrors ${NPM_SCRIPT_COMMAND}.log)
+  if [[ ! -z "$error" ]]; then
+    echo -e "${FG_BOLD_WHITE}${BG_RED} FLAKE DETECTED: ${FG_RED} ${error} ${NC}"
+
+    # use GitHub Action annotation to highlight flake
+    echo -e "::warning::FLAKE DETECTED: ${error}"
+    echo -e "${FG_BOLD_WHITE}${BG_BLUE} RETRYING: ${NC} ${FG_GRAY}(${i} of ${NUM_RETRIES})${FG_BLUE} ${COMMAND_TO_RUN}${NC}";
+  else
+    break
+  fi
+done