[FTR] Improve running FTR tests by avoiding rerunning a config when agent is lost

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -52,6 +52,15 @@ while read -r config; do
     continue;
   fi
 
+  echo "--- Checking for existing scout event for config $config"
+  # We do not use --dontFailOnError here because we need a real exit code to determine if we should skip
+  if node scripts/scout check-event --buildNumber "${BUILDKITE_BUILD_NUMBER:-}" --testConfig "$config"; then
+    echo "--- Scout event found for config $config, skipping"
+    continue
+  else
+    echo "--- Scout event not found for config $config, running tests"
+  fi
+
   FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS"
   echo "--- $ $FULL_COMMAND"
 
@@ -80,6 +89,22 @@ while read -r config; do
   lastCode=$?
   set -e;
 
+  # Scout reporter
+  REPORT_DIR=$(ls -td .scout/reports/scout-ftr-*/ 2>/dev/null | head -n 1)
+  if [ -n "$REPORT_DIR" ]; then
+    EVENT_LOG_FILE="${REPORT_DIR}event-log.ndjson"
+    if [ -f "$EVENT_LOG_FILE" ]; then
+      # Upload events after running each config
+      export SCOUT_EVENT_LOG_PATH="$EVENT_LOG_FILE"
+      source .buildkite/scripts/steps/test/scout_upload_report_events.sh
+      unset SCOUT_EVENT_LOG_PATH
+    else
+      echo "Could not find event log file '$EVENT_LOG_FILE' for config $config"
+    fi
+  else
+    echo "Could not find any scout report directory '$EVENT_LOG_FILE'."
+  fi
+
   timeSec=$(($(date +%s)-start))
   if [[ $timeSec -gt 60 ]]; then
     min=$((timeSec/60))
@@ -114,7 +139,4 @@ echo "--- FTR configs complete"
 printf "%s\n" "${results[@]}"
 echo ""
 
-# Scout reporter
-source .buildkite/scripts/steps/test/scout_upload_report_events.sh
-
 exit $exitCode

.buildkite/scripts/steps/test/scout_upload_report_events.sh

@@ -2,9 +2,15 @@
 
 set -euo pipefail
 
+SCOUT_EXTRA_ARGS=""
+if [ -n "${SCOUT_EVENT_LOG_PATH:-}" ]; then
+  SCOUT_EXTRA_ARGS="--eventLogPath $SCOUT_EVENT_LOG_PATH"
+fi
+
 echo "--- Upload Scout reporter events to AppEx QA's team cluster"
 if [[ "${SCOUT_REPORTER_ENABLED:-}" == "true" ]]; then
-  node scripts/scout upload-events --dontFailOnError
+  # shellcheck disable=SC2086
+  node scripts/scout upload-events --dontFailOnError $SCOUT_EXTRA_ARGS
 else
   echo "⚠️ The SCOUT_REPORTER_ENABLED environment variable is not 'true'. Skipping event upload."
 fi

src/platform/packages/private/kbn-scout-reporting/src/cli/check_event.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Command } from '@kbn/dev-cli-runner';
+import {
+  SCOUT_REPORTER_ES_URL,
+  SCOUT_REPORTER_ES_API_KEY,
+  SCOUT_REPORTER_ES_VERIFY_CERTS,
+  SCOUT_TEST_EVENTS_DATA_STREAM_NAME,
+} from '@kbn/scout-info';
+import { getValidatedESClient } from '../helpers/elasticsearch';
+
+export const checkEventCommand: Command<void> = {
+  name: 'check-event',
+  description:
+    'Check if a specific event exists in Elasticsearch for a given build and test config',
+  flags: {
+    string: ['buildNumber', 'testConfig', 'esURL', 'esAPIKey'],
+    boolean: ['verifyTLSCerts', 'dontFailOnError'],
+    default: {
+      esURL: SCOUT_REPORTER_ES_URL,
+      esAPIKey: SCOUT_REPORTER_ES_API_KEY,
+      verifyTLSCerts: SCOUT_REPORTER_ES_VERIFY_CERTS,
+      dontFailOnError: false,
+    },
+    help: `
+    --buildNumber     (required)  The Buildkite build number.
+    --testConfig      (required)  The path to the test config file.
+    --esURL           (required)  Elasticsearch URL [env: SCOUT_REPORTER_ES_URL]
+    --esAPIKey        (required)  Elasticsearch API Key [env: SCOUT_REPORTER_ES_API_KEY]
+    --verifyTLSCerts  (optional)  Verify TLS certificates [env: SCOUT_REPORTER_ES_VERIFY_CERTS]
+    --dontFailOnError (optional)  If present, errors will be logged but the process will not fail (default: false)
+    `,
+  },
+  run: async ({ flagsReader, log }) => {
+    const buildNumber = flagsReader.requiredString('buildNumber');
+    const testConfig = flagsReader.requiredString('testConfig');
+    const esURL = flagsReader.requiredString('esURL');
+    const esAPIKey = flagsReader.requiredString('esAPIKey');
+    const verifyTLSCerts = flagsReader.boolean('verifyTLSCerts');
+    const dontFailOnError = flagsReader.boolean('dontFailOnError');
+
+    log.info(`Connecting to Elasticsearch at ${esURL}`);
+    const es = await getValidatedESClient(
+      {
+        node: esURL,
+        auth: { apiKey: esAPIKey },
+        tls: {
+          rejectUnauthorized: verifyTLSCerts,
+        },
+      },
+      { log, cli: true }
+    );
+
+    log.info(`Checking for events with buildNumber: ${buildNumber} and testConfig: ${testConfig}`);
+
+    try {
+      const result = await es.search({
+        index: SCOUT_TEST_EVENTS_DATA_STREAM_NAME,
+        size: 0,
+        query: {
+          bool: {
+            filter: [
+              {
+                range: {
+                  '@timestamp': {
+                    gte: 'now-1d/d',
+                    lte: 'now',
+                  },
+                },
+              },
+            ],
+            must: [
+              { term: { 'buildkite.build.number': buildNumber } },
+              { term: { 'test_run.config.file.path': testConfig } },
+            ],
+          },
+        },
+      });
+
+      if ((result.hits.total as any).value > 0) {
+        log.success(`Found events for buildNumber: ${buildNumber} and testConfig: ${testConfig}`);
+      } else {
+        log.info(
+          `No events was found for buildNumber: ${buildNumber} and testConfig: ${testConfig}`
+        );
+        if (!dontFailOnError) {
+          process.exitCode = 1;
+        }
+      }
+    } catch (error) {
+      log.error(
+        `Error checking events for buildNumber: ${buildNumber} and testConfig: ${testConfig}`
+      );
+      log.error(error);
+      if (!dontFailOnError) {
+        process.exitCode = 1;
+      }
+    }
+  },
+};

src/platform/packages/private/kbn-scout-reporting/src/cli/index.ts

@@ -10,3 +10,4 @@
 export { initializeReportDatastream } from './initialize_report_datastream';
 export { uploadEvents } from './upload_events';
 export { updateTestConfigStats } from './update_test_config_stats';
+export { checkEventCommand } from './check_event';

src/platform/packages/shared/kbn-scout/src/cli/index.ts

@@ -27,6 +27,7 @@ export async function run() {
       reportingCLI.initializeReportDatastream,
       reportingCLI.uploadEvents,
       reportingCLI.updateTestConfigStats,
+      reportingCLI.checkEventCommand,
       createTestTrack,
     ]
   ).execute();