[8.x] [Security Solution][Endpoint] Cypress test improvements to capture Agent diagnostics file when test fails (#202965)

x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts

@@ -11,6 +11,7 @@
 
 import type { CasePostRequest } from '@kbn/cases-plugin/common/api';
 import type { UsageRecord } from '@kbn/security-solution-serverless/server/types';
+import type { HostVmTransferResponse } from '../../../scripts/endpoint/common/types';
 import type {
   DeletedEndpointHeartbeats,
   IndexedEndpointHeartbeats,
@@ -30,6 +31,7 @@ import type {
   UninstallAgentFromHostTaskOptions,
   IsAgentAndEndpointUninstalledFromHostTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from './types';
 import type {
   DeleteIndexedFleetEndpointPoliciesResponse,
@@ -267,6 +269,12 @@ declare global {
         arg: LogItTaskOptions,
         options?: Partial<Loggable & Timeoutable>
       ): Chainable<null>;
+
+      task(
+        name: 'captureHostVmAgentDiagnostics',
+        arg: CaptureHostVmAgentDiagnosticsOptions,
+        options?: Partial<Loggable & Timeoutable>
+      ): Chainable<Omit<HostVmTransferResponse, 'delete'>>;
     }
   }
 }

x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts

@@ -67,6 +67,15 @@ describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => {
       }
     });
 
+    afterEach(function () {
+      if (Cypress.env('IS_CI') && this.currentTest?.isFailed() && createdHost) {
+        cy.task('captureHostVmAgentDiagnostics', {
+          hostname: createdHost.hostname,
+          fileNamePrefix: this.currentTest?.fullTitle(),
+        });
+      }
+    });
+
     it('"get-file --path" - should retrieve a file', () => {
       const downloadsFolder = Cypress.config('downloadsFolder');
 

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -11,6 +11,10 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common';
 import execa from 'execa';
 import type { KbnClient } from '@kbn/test';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/repo-info';
+// This is a Cypress module and only used by Cypress, so disabling "should" be safe
+// eslint-disable-next-line import/no-nodejs-modules
+import { mkdir } from 'node:fs/promises';
 import type { IndexedEndpointHeartbeats } from '../../../../common/endpoint/data_loaders/index_endpoint_hearbeats';
 import {
   deleteIndexedEndpointHeartbeats,
@@ -53,6 +57,7 @@ import type {
   LoadUserAndRoleCyTaskOptions,
   CreateUserAndRoleCyTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from '../types';
 import type {
   DeletedIndexedEndpointRuleAlerts,
@@ -75,6 +80,7 @@ import {
   deleteAgentPolicy,
   fetchAgentPolicyEnrollmentKey,
   getOrCreateDefaultAgentPolicy,
+  setAgentLoggingLevel,
 } from '../../../../scripts/endpoint/common/fleet_services';
 import { startElasticAgentWithDocker } from '../../../../scripts/endpoint/common/elastic_agent_service';
 import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
@@ -433,6 +439,7 @@ ${s1Info.status}
                   log,
                   kbnClient,
                 });
+            await setAgentLoggingLevel(kbnClient, newHost.agentId, 'debug', log);
             await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000);
             return newHost;
           } catch (err) {
@@ -531,5 +538,65 @@ ${s1Info.status}
       await startEndpointHost(hostName);
       return null;
     },
+
+    /**
+     * Generates an Agent Diagnostics archive (ZIP) directly on the Host VM and saves it to a directory
+     * that is then included with the list of Artifacts that are captured with Buildkite job.
+     *
+     * ### Usage:
+     *
+     * This task is best used from a `afterEach()` by checking if the test failed and if so (and it
+     * was a test that was running against a host VM), then capture the diagnostics file
+     *
+     * @param hostname
+     *
+     * @example
+     *
+     * describe('something', () => {
+     *   let hostVm;
+     *
+     *   afterEach(function() { // << Important: Note the use of `function()` here instead of arrow function
+     *     if (this.currentTest?.isFailed() && hostVm) {
+     *       cy.task('captureHostVmAgentDiagnostics', { hostname: hostVm.hostname });
+     *     }
+     *   });
+     *
+     *   //...
+     * })
+     */
+    captureHostVmAgentDiagnostics: async ({
+      hostname,
+      fileNamePrefix = '',
+    }: CaptureHostVmAgentDiagnosticsOptions) => {
+      const { log } = await stackServicesPromise;
+
+      log.info(`Capturing agent diagnostics for host VM [${hostname}]`);
+
+      const vmClient = getHostVmClient(hostname, undefined, undefined, log);
+      const fileName = `elastic-agent-diagnostics-${hostname}-${new Date()
+        .toISOString()
+        .replace(/:/g, '.')}.zip`;
+      const vmDiagnosticsFile = `/tmp/${fileName}`;
+      const localDiagnosticsDir = `${REPO_ROOT}/target/test_failures`;
+      const localDiagnosticsFile = `${localDiagnosticsDir}/${
+        fileNamePrefix
+          ? // Insure the file name prefix does not have characters that can't be used in file names
+            `${fileNamePrefix.replace(/[><:"/\\|?*'`{} ]/g, '_')}-`
+          : ''
+      }${fileName}`;
+
+      await mkdir(localDiagnosticsDir, { recursive: true });
+
+      // generate diagnostics file on the host and then download it
+      await vmClient.exec(
+        `sudo /opt/Elastic/Agent/elastic-agent diagnostics --file ${vmDiagnosticsFile}`
+      );
+      return vmClient.download(vmDiagnosticsFile, localDiagnosticsFile).then((response) => {
+        log.info(`Agent diagnostic file for host [${hostname}] has been downloaded and is available at:
+  ${response.filePath}
+`);
+        return { filePath: response.filePath };
+      });
+    },
   });
 };

x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts

@@ -16,10 +16,20 @@ export const setupToolingLogLevel = (config: Cypress.PluginConfigOptions) => {
   const log = createToolingLogger();
   const defaultToolingLogLevel = config.env.TOOLING_LOG_LEVEL;
 
-  log.info(`Cypress config 'env.TOOLING_LOG_LEVEL': ${defaultToolingLogLevel}`);
+  log.info(`
+
+Cypress Configuration File: ${config.configFile}
+
+'env.TOOLING_LOG_LEVEL' set to: ${defaultToolingLogLevel}
+
+*** FYI: ***  To help with test failures, an environmental variable named 'TOOLING_LOG_LEVEL' can be set
+              with a value of 'verbose' in order to capture more data in the logs. This environment
+              property can be set either in the runtime environment (ex. local shell or buildkite) or
+              directly in the Cypress configuration file \`env: {}\` section.
+
+  `);
 
   if (defaultToolingLogLevel && defaultToolingLogLevel !== createToolingLogger.defaultLogLevel) {
     createToolingLogger.defaultLogLevel = defaultToolingLogLevel;
-    log.info(`Default log level for 'createToolingLogger()' set to ${defaultToolingLogLevel}`);
   }
 };

x-pack/plugins/security_solution/public/management/cypress/types.ts

@@ -81,3 +81,8 @@ export interface LogItTaskOptions {
   level: keyof Pick<ToolingLog, 'info' | 'debug' | 'verbose'>;
   data: any;
 }
+
+export interface CaptureHostVmAgentDiagnosticsOptions {
+  hostname: string;
+  fileNamePrefix?: string;
+}

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts

@@ -277,17 +277,19 @@ const startFleetServerWithDocker = async ({
   const isServerless = await isServerlessKibanaFlavor(kbnClient);
   const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient));
   const containerName = `dev-fleet-server.${port}`;
+  let fleetServerVersionInfo = '';
 
   log.info(
     `Starting a new fleet server using Docker\n    Agent version: ${agentVersion}\n    Server URL: ${fleetServerUrl}`
   );
 
   let retryAttempt = isServerless ? 0 : 1;
   const attemptServerlessFleetServerSetup = async (): Promise<StartedServer> => {
+    fleetServerVersionInfo = '';
+
     return log.indent(4, async () => {
       const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`;
       let containerId = '';
-      let fleetServerVersionInfo = '';
 
       if (isLocalhost(esURL.hostname)) {
         esURL.hostname = localhostRealIp;
@@ -361,8 +363,17 @@ const startFleetServerWithDocker = async ({
         }
 
         fleetServerVersionInfo = isServerless
-          ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument
-            'Running latest standalone fleet server'
+          ? (
+              await execa
+                .command(`docker exec ${containerName} /usr/bin/fleet-server --version`)
+                .catch((err) => {
+                  log.verbose(
+                    `Failed to retrieve fleet-server (serverless/standalone) version information from running instance.`,
+                    err
+                  );
+                  return { stdout: 'Unable to retrieve version information (serverless)' };
+                })
+            ).stdout
           : (
               await execa('docker', [
                 'exec',
@@ -424,7 +435,7 @@ Kill container:       ${chalk.cyan(`docker kill ${containerId}`)}
 
   const response: StartedServer = await attemptServerlessFleetServerSetup();
 
-  log.info(`Done. Fleet server up and running`);
+  log.info(`Done. Fleet server up and running (version: ${fleetServerVersionInfo})`);
 
   return response;
 };