[Security Solution][Endpoint] Cypress test improvements to capture Agent diagnostics file when test fails

x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts

@@ -11,6 +11,7 @@
 
 import type { CasePostRequest } from '@kbn/cases-plugin/common/api';
 import type { UsageRecord } from '@kbn/security-solution-serverless/server/types';
+import type { HostVmTransferResponse } from '../../../scripts/endpoint/common/types';
 import type {
   DeletedEndpointHeartbeats,
   IndexedEndpointHeartbeats,
@@ -30,6 +31,7 @@ import type {
   UninstallAgentFromHostTaskOptions,
   IsAgentAndEndpointUninstalledFromHostTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from './types';
 import type {
   DeleteIndexedFleetEndpointPoliciesResponse,
@@ -267,6 +269,12 @@ declare global {
         arg: LogItTaskOptions,
         options?: Partial<Loggable & Timeoutable>
       ): Chainable<null>;
+
+      task(
+        name: 'captureHostVmAgentDiagnostics',
+        arg: CaptureHostVmAgentDiagnosticsOptions,
+        options?: Partial<Loggable & Timeoutable>
+      ): Chainable<Omit<HostVmTransferResponse, 'delete'>>;
     }
   }
 }

x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts

@@ -66,6 +66,15 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => {
       }
     });
 
+    afterEach(function () {
+      if (Cypress.env('IS_CI') && this.currentTest?.isFailed() && createdHost) {
+        cy.task('captureHostVmAgentDiagnostics', {
+          hostname: createdHost.hostname,
+          fileNamePrefix: this.currentTest?.fullTitle(),
+        });
+      }
+    });
+
     it('"get-file --path" - should retrieve a file', () => {
       const downloadsFolder = Cypress.config('downloadsFolder');
 

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -11,6 +11,10 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common';
 import execa from 'execa';
 import type { KbnClient } from '@kbn/test';
 import type { ToolingLog } from '@kbn/tooling-log';
+import { REPO_ROOT } from '@kbn/repo-info';
+// This is a Cypress module and only used by Cypress, so disabling "should" be safe
+// eslint-disable-next-line import/no-nodejs-modules
+import { mkdir } from 'node:fs/promises';
 import type { IndexedEndpointHeartbeats } from '../../../../common/endpoint/data_loaders/index_endpoint_hearbeats';
 import {
   deleteIndexedEndpointHeartbeats,
@@ -53,6 +57,7 @@ import type {
   LoadUserAndRoleCyTaskOptions,
   CreateUserAndRoleCyTaskOptions,
   LogItTaskOptions,
+  CaptureHostVmAgentDiagnosticsOptions,
 } from '../types';
 import type {
   DeletedIndexedEndpointRuleAlerts,
@@ -75,6 +80,7 @@ import {
   deleteAgentPolicy,
   fetchAgentPolicyEnrollmentKey,
   getOrCreateDefaultAgentPolicy,
+  setAgentLoggingLevel,
 } from '../../../../scripts/endpoint/common/fleet_services';
 import { startElasticAgentWithDocker } from '../../../../scripts/endpoint/common/elastic_agent_service';
 import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy';
@@ -433,6 +439,7 @@ ${s1Info.status}
                   log,
                   kbnClient,
                 });
+            await setAgentLoggingLevel(kbnClient, newHost.agentId, 'debug', log);
             await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000);
             return newHost;
           } catch (err) {
@@ -531,5 +538,65 @@ ${s1Info.status}
       await startEndpointHost(hostName);
       return null;
     },
+
+    /**
+     * Generates an Agent Diagnostics archive (ZIP) directly on the Host VM and saves it to a directory
+     * that is then included with the list of Artifacts that are captured with Buildkite job.
+     *
+     * ### Usage:
+     *
+     * This task is best used from a `afterEach()` by checking if the test failed and if so (and it
+     * was a test that was running against a host VM), then capture the diagnostics file
+     *
+     * @param hostname
+     *
+     * @example
+     *
+     * describe('something', () => {
+     *   let hostVm;
+     *
+     *   afterEach(function() { // << Important: Note the use of `function()` here instead of arrow function
+     *     if (this.currentTest?.isFailed() && hostVm) {
+     *       cy.task('captureHostVmAgentDiagnostics', { hostname: hostVm.hostname });
+     *     }
+     *   });
+     *
+     *   //...
+     * })
+     */
+    captureHostVmAgentDiagnostics: async ({
+      hostname,
+      fileNamePrefix = '',
+    }: CaptureHostVmAgentDiagnosticsOptions) => {
+      const { log } = await stackServicesPromise;
+
+      log.info(`Capturing agent diagnostics for host VM [${hostname}]`);
+
+      const vmClient = getHostVmClient(hostname, undefined, undefined, log);
+      const fileName = `elastic-agent-diagnostics-${hostname}-${new Date()
+        .toISOString()
+        .replace(/:/g, '.')}.zip`;
+      const vmDiagnosticsFile = `/tmp/${fileName}`;
+      const localDiagnosticsDir = `${REPO_ROOT}/target/test_failures`;
+      const localDiagnosticsFile = `${localDiagnosticsDir}/${
+        fileNamePrefix
+          ? // Insure the file name prefix does not have characters that can't be used in file names
+            `${fileNamePrefix.replace(/[><:"/\\|?*'`{} ]/g, '_')}-`
+          : ''
+      }${fileName}`;
+
+      await mkdir(localDiagnosticsDir, { recursive: true });
+
+      // generate diagnostics file on the host and then download it
+      await vmClient.exec(
+        `sudo /opt/Elastic/Agent/elastic-agent diagnostics --file ${vmDiagnosticsFile}`
+      );
+      return vmClient.download(vmDiagnosticsFile, localDiagnosticsFile).then((response) => {
+        log.info(`Agent diagnostic file for host [${hostname}] has been downloaded and is available at:
+  ${response.filePath}
+`);
+        return { filePath: response.filePath };
+      });
+    },
   });
 };

x-pack/plugins/security_solution/public/management/cypress/support/setup_tooling_log_level.ts

@@ -16,10 +16,20 @@ export const setupToolingLogLevel = (config: Cypress.PluginConfigOptions) => {
   const log = createToolingLogger();
   const defaultToolingLogLevel = config.env.TOOLING_LOG_LEVEL;
 
-  log.info(`Cypress config 'env.TOOLING_LOG_LEVEL': ${defaultToolingLogLevel}`);
+  log.info(`
+
+Cypress Configuration File: ${config.configFile}
+
+'env.TOOLING_LOG_LEVEL' set to: ${defaultToolingLogLevel}
+
+*** FYI: ***  To help with test failures, an environmental variable named 'TOOLING_LOG_LEVEL' can be set
+              with a value of 'verbose' in order to capture more data in the logs. This environment
+              property can be set either in the runtime environment (ex. local shell or buildkite) or
+              directly in the Cypress configuration file \`env: {}\` section.
+
+  `);
 
   if (defaultToolingLogLevel && defaultToolingLogLevel !== createToolingLogger.defaultLogLevel) {
     createToolingLogger.defaultLogLevel = defaultToolingLogLevel;
-    log.info(`Default log level for 'createToolingLogger()' set to ${defaultToolingLogLevel}`);
   }
 };

x-pack/plugins/security_solution/public/management/cypress/types.ts

@@ -81,3 +81,8 @@ export interface LogItTaskOptions {
   level: keyof Pick<ToolingLog, 'info' | 'debug' | 'verbose'>;
   data: any;
 }
+
+export interface CaptureHostVmAgentDiagnosticsOptions {
+  hostname: string;
+  fileNamePrefix?: string;
+}

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts

@@ -277,17 +277,19 @@ const startFleetServerWithDocker = async ({
   const isServerless = await isServerlessKibanaFlavor(kbnClient);
   const esURL = new URL(await getFleetElasticsearchOutputHost(kbnClient));
   const containerName = `dev-fleet-server.${port}`;
+  let fleetServerVersionInfo = '';
 
   log.info(
     `Starting a new fleet server using Docker\n    Agent version: ${agentVersion}\n    Server URL: ${fleetServerUrl}`
   );
 
   let retryAttempt = isServerless ? 0 : 1;
   const attemptServerlessFleetServerSetup = async (): Promise<StartedServer> => {
+    fleetServerVersionInfo = '';
+
     return log.indent(4, async () => {
       const hostname = `dev-fleet-server.${port}.${Math.random().toString(32).substring(2, 6)}`;
       let containerId = '';
-      let fleetServerVersionInfo = '';
 
       if (isLocalhost(esURL.hostname)) {
         esURL.hostname = localhostRealIp;
@@ -361,8 +363,17 @@ const startFleetServerWithDocker = async ({
         }
 
         fleetServerVersionInfo = isServerless
-          ? // `/usr/bin/fleet-server` process does not seem to support a `--version` type of argument
-            'Running latest standalone fleet server'
+          ? (
+              await execa
+                .command(`docker exec ${containerName} /usr/bin/fleet-server --version`)
+                .catch((err) => {
+                  log.verbose(
+                    `Failed to retrieve fleet-server (serverless/standalone) version information from running instance.`,
+                    err
+                  );
+                  return { stdout: 'Unable to retrieve version information (serverless)' };
+                })
+            ).stdout
           : (
               await execa('docker', [
                 'exec',
@@ -424,7 +435,7 @@ Kill container:       ${chalk.cyan(`docker kill ${containerId}`)}
 
   const response: StartedServer = await attemptServerlessFleetServerSetup();
 
-  log.info(`Done. Fleet server up and running`);
+  log.info(`Done. Fleet server up and running (version: ${fleetServerVersionInfo})`);
 
   return response;
 };

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -56,12 +56,14 @@ import type {
   DeleteAgentPolicyResponse,
   EnrollmentAPIKey,
   GenerateServiceTokenResponse,
+  GetActionStatusResponse,
   GetAgentsRequest,
   GetEnrollmentAPIKeysResponse,
   GetOutputsResponse,
   PostAgentUnenrollResponse,
   UpdateAgentPolicyRequest,
   UpdateAgentPolicyResponse,
+  PostNewAgentActionResponse,
 } from '@kbn/fleet-plugin/common/types';
 import semver from 'semver';
 import axios from 'axios';
@@ -1499,3 +1501,92 @@ export const updateAgentPolicy = async (
     .catch(catchAxiosErrorFormatAndThrow)
     .then((response) => response.data.item);
 };
+
+/**
+ * Sets the log level on a Fleet agent and waits a bit of time to allow it for to
+ * complete (but does not error if it does not complete)
+ *
+ * @param kbnClient
+ * @param agentId
+ * @param logLevel
+ * @param log
+ */
+export const setAgentLoggingLevel = async (
+  kbnClient: KbnClient,
+  agentId: string,
+  logLevel: 'debug' | 'info' | 'warning' | 'error',
+  log: ToolingLog = createToolingLogger()
+): Promise<PostNewAgentActionResponse> => {
+  log.debug(`Setting fleet agent [${agentId}] logging level to [${logLevel}]`);
+
+  const response = await kbnClient
+    .request<PostNewAgentActionResponse>({
+      method: 'POST',
+      path: `/api/fleet/agents/${agentId}/actions`,
+      body: { action: { type: 'SETTINGS', data: { log_level: logLevel } } },
+      headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 },
+    })
+    .then((res) => res.data);
+
+  // Wait to see if the action completes, but don't `throw` if it does not
+  await waitForFleetAgentActionToComplete(kbnClient, response.item.id)
+    .then(() => {
+      log.debug(`Fleet action to set agent [${agentId}] logging level to [${logLevel}] completed!`);
+    })
+    .catch((err) => {
+      log.debug(err.message);
+    });
+
+  return response;
+};
+
+/**
+ * Retrieve fleet agent action statuses
+ * @param kbnClient
+ */
+export const fetchFleetAgentActionStatus = async (
+  kbnClient: KbnClient
+): Promise<GetActionStatusResponse> => {
+  return kbnClient
+    .request<GetActionStatusResponse>({
+      method: 'GET',
+      path: agentRouteService.getActionStatusPath(),
+      query: { perPage: 1000 },
+      headers: { 'Elastic-Api-Version': API_VERSIONS.public.v1 },
+    })
+    .then((response) => response.data);
+};
+
+/**
+ * Check and wait until a Fleet Agent action is complete.
+ * @param kbnClient
+ * @param actionId
+ * @param timeout
+ *
+ * @throws
+ */
+export const waitForFleetAgentActionToComplete = async (
+  kbnClient: KbnClient,
+  actionId: string,
+  timeout: number = 20_000
+): Promise<void> => {
+  await pRetry(
+    async (attempts) => {
+      const { items: actionList } = await fetchFleetAgentActionStatus(kbnClient);
+      const actionInfo = actionList.find((action) => action.actionId === actionId);
+
+      if (!actionInfo) {
+        throw new Error(
+          `Fleet Agent action id [${actionId}] was not found in list of actions retrieved from fleet!`
+        );
+      }
+
+      if (actionInfo.status === 'IN_PROGRESS') {
+        throw new Error(
+          `Fleet agent action id [${actionId}] remains in progress after [${attempts}] attempts to check its status`
+        );
+      }
+    },
+    { maxTimeout: 2_000, maxRetryTime: timeout }
+  );
+};

x-pack/plugins/security_solution/scripts/endpoint/common/types.ts

@@ -14,8 +14,12 @@ export interface HostVm {
   exec: (command: string) => Promise<HostVmExecResponse>;
   mount: (localDir: string, hostVmDir: string) => Promise<HostVmMountResponse>;
   unmount: (hostVmDir: string) => Promise<void>;
-  /** Uploads/copies a file from the local machine to the VM */
+  /** @deprecated use `upload` */
   transfer: (localFilePath: string, destFilePath: string) => Promise<HostVmTransferResponse>;
+  /** Uploads/copies a file from the local machine to the VM */
+  upload: (localFilePath: string, destFilePath: string) => Promise<HostVmTransferResponse>;
+  /** Downloads a file from the host VM to the local machine */
+  download: (vmFilePath: string, localFilePath: string) => Promise<HostVmTransferResponse>;
   destroy: () => Promise<void>;
   info: () => string;
   stop: () => void;
@@ -33,8 +37,8 @@ export interface HostVmMountResponse {
   unmount: () => Promise<void>;
 }
 export interface HostVmTransferResponse {
-  /** The file path of the file on the host vm */
+  /** The path of the file that was either uploaded to the host VM or downloaded to the local machine from the VM */
   filePath: string;
-  /** Delete the file from the host VM */
+  /** Delete the file from the host VM or from the local machine depending on what client method was used */
   delete: () => Promise<HostVmExecResponse>;
 }