[CI] Improve running FTR tests by avoiding rerunning a config when agent is lost (reapplied) (#238631)

## Summary
Reapply @maryam-saeidi's #237705 

+ Adds a fix for when the scout reporter is not enabled

Author: delanni

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -53,7 +53,16 @@ while read -r config; do
   fi
 
   FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS"
-  echo "--- $ $FULL_COMMAND"
+
+  CONFIG_EXECUTION_KEY="${config}_executed"
+  IS_CONFIG_EXECUTION=$(buildkite-agent meta-data get "$CONFIG_EXECUTION_KEY" --default "false" --log-level error)
+
+  if [[ "${IS_CONFIG_EXECUTION}" == "true" ]]; then
+    echo "--- [ already-tested ] $FULL_COMMAND"
+    continue
+  else
+    echo "--- $ $FULL_COMMAND"
+  fi
 
   start=$(date +%s)
 
@@ -80,6 +89,17 @@ while read -r config; do
   lastCode=$?
   set -e;
 
+  # Scout reporter
+  if [[ "${SCOUT_REPORTER_ENABLED:-}" =~ ^(1|true)$ ]]; then
+    # Upload events after running each config
+    echo "Upload Scout reporter events to AppEx QA's team cluster for config $config"
+    node scripts/scout upload-events --dontFailOnError
+    echo "Upload successful, removing local events at .scout/reports"
+    rm -rf .scout/reports
+  else
+    echo "SCOUT_REPORTER_ENABLED=$SCOUT_REPORTER_ENABLED, skipping event upload."
+  fi
+
   timeSec=$(($(date +%s)-start))
   if [[ $timeSec -gt 60 ]]; then
     min=$((timeSec/60))
@@ -93,7 +113,10 @@ while read -r config; do
     duration: ${duration}
     result: ${lastCode}")
 
-  if [ $lastCode -ne 0 ]; then
+  if [ $lastCode -eq 0 ]; then
+    # Test was successful, so mark it as executed
+    buildkite-agent meta-data set "$CONFIG_EXECUTION_KEY" "true"
+  else
     exitCode=10
     echo "FTR exited with code $lastCode"
     echo "^^^ +++"
@@ -114,7 +137,4 @@ echo "--- FTR configs complete"
 printf "%s\n" "${results[@]}"
 echo ""
 
-# Scout reporter
-source .buildkite/scripts/steps/test/scout_upload_report_events.sh
-
 exit $exitCode