[CI] Harden Defend Workflows VM provisioning

.buildkite/scripts/steps/functional/defend_workflows.sh

@@ -7,6 +7,8 @@ source .buildkite/scripts/steps/functional/common.sh
 export JOB=kibana-defend-workflows-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
+source .buildkite/scripts/steps/functional/ensure_virtualbox.sh
+
 echo "--- Defend Workflows Cypress tests"
 
 cd x-pack/solutions/security/plugins/security_solution

.buildkite/scripts/steps/functional/defend_workflows_burn.sh

@@ -9,6 +9,8 @@ export JOB=kibana-defend-workflows-cypress
 
 buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false'
 
+source .buildkite/scripts/steps/functional/ensure_virtualbox.sh
+
 echo "--- Defend Workflows Cypress tests, burning changed specs (Chrome)"
 
 yarn --cwd x-pack/solutions/security/plugins/security_solution cypress:changed-specs-only

.buildkite/scripts/steps/functional/defend_workflows_serverless.sh

@@ -7,6 +7,8 @@ source .buildkite/scripts/steps/functional/common.sh
 export JOB=kibana-defend-workflows-serverless-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
+source .buildkite/scripts/steps/functional/ensure_virtualbox.sh
+
 echo "--- Defend Workflows Cypress tests on Serverless"
 
 cd x-pack/solutions/security/plugins/security_solution

.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh

@@ -9,6 +9,8 @@ export JOB=kibana-defend-workflows-serverless-cypress
 
 buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false'
 
+source .buildkite/scripts/steps/functional/ensure_virtualbox.sh
+
 echo "--- Defend Workflows Cypress tests, burning changed specs (Chrome)"
 
 yarn --cwd x-pack/solutions/security/plugins/security_solution cypress:dw:serverless:changed-specs-only

.buildkite/scripts/steps/functional/ensure_virtualbox.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+# Ensures VirtualBox is installed with working kernel modules and
+# exports VAGRANT_DEFAULT_PROVIDER=virtualbox.
+# Must be sourced (not executed) so the export propagates to the caller.
+
+echo "--- Ensure VirtualBox provider"
+
+_vbox_kernel_ok() {
+  local output
+  output=$(VBoxManage --version 2>&1)
+  if echo "$output" | grep -qi "kernel module is not loaded"; then
+    return 1
+  fi
+  return 0
+}
+
+_ensure_vbox_modules() {
+  sudo modprobe vboxdrv 2>/dev/null && return 0
+  sudo /sbin/vboxconfig 2>/dev/null && return 0
+  sudo /sbin/rcvboxdrv setup 2>/dev/null && return 0
+  return 1
+}
+
+_print_vbox_diagnostics() {
+  echo "VirtualBox diagnostics:"
+  echo "  vagrant: $(vagrant --version 2>/dev/null || echo 'not found')"
+  echo "  VBoxManage: $(which VBoxManage 2>/dev/null || echo 'not in PATH')"
+  echo "  VBoxManage --version: $(VBoxManage --version 2>&1 || echo 'failed')"
+  echo "  vbox modules: $(lsmod 2>/dev/null | grep vbox || echo 'none loaded')"
+  echo "  vbox packages: $(dpkg -l 2>/dev/null | grep -i virtualbox | awk '{print $2, $3}' || echo 'none')"
+  echo "  kernel: $(uname -r)"
+  echo "  dkms status: $(dkms status 2>/dev/null || echo 'dkms not available')"
+}
+
+_upgrade_to_vbox71() {
+  echo "Upgrading to VirtualBox 7.1 (supports newer kernels)..."
+
+  # Oracle's repo is pre-configured on CI images; install 7.1 from there
+  if apt-cache show virtualbox-7.1 &>/dev/null; then
+    echo "virtualbox-7.1 available from Oracle repo, installing..."
+    sudo apt-get install -y --no-install-recommends virtualbox-7.1 2>&1
+    return $?
+  fi
+
+  # If Oracle repo isn't configured, add it
+  echo "Adding Oracle VirtualBox repository..."
+  local codename
+  codename=$(lsb_release -cs 2>/dev/null || echo "noble")
+  wget -qO- https://www.virtualbox.org/download/oracle_vbox_2016.asc | sudo gpg --dearmor --yes -o /usr/share/keyrings/oracle-virtualbox-2016.gpg
+  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/oracle-virtualbox-2016.gpg] https://download.virtualbox.org/virtualbox/debian ${codename} contrib" | \
+    sudo tee /etc/apt/sources.list.d/virtualbox-oracle.list
+  sudo apt-get update -qq
+  sudo apt-get install -y --no-install-recommends virtualbox-7.1 2>&1
+  return $?
+}
+
+_ensure_virtualbox() {
+  # 1. Already working — nothing to do
+  if command -v VBoxManage &>/dev/null && _vbox_kernel_ok; then
+    echo "VirtualBox ready: $(VBoxManage --version 2>&1 | tail -1)"
+    return 0
+  fi
+
+  # 2. VBoxManage exists but kernel module not loaded — try loading
+  if command -v VBoxManage &>/dev/null; then
+    echo "VBoxManage found but kernel module not loaded, attempting recovery..."
+    if _ensure_vbox_modules && _vbox_kernel_ok; then
+      echo "VirtualBox kernel module recovered: $(VBoxManage --version 2>&1 | tail -1)"
+      return 0
+    fi
+
+    # Modules couldn't load — likely kernel too new for current VBox version.
+    # Upgrade to 7.1 which supports newer kernels.
+    echo "Kernel module build failed for current VirtualBox, upgrading to 7.1..."
+    if _upgrade_to_vbox71 && _ensure_vbox_modules && _vbox_kernel_ok; then
+      echo "VirtualBox 7.1 installed and ready: $(VBoxManage --version 2>&1 | tail -1)"
+      return 0
+    fi
+  else
+    # 3. VBoxManage not found at all — install 7.1 from scratch
+    echo "VirtualBox not found, installing 7.1..."
+    sudo apt-get update -qq
+    if _upgrade_to_vbox71 && _ensure_vbox_modules && _vbox_kernel_ok; then
+      echo "VirtualBox 7.1 installed and ready: $(VBoxManage --version 2>&1 | tail -1)"
+      return 0
+    fi
+  fi
+
+  _print_vbox_diagnostics
+  echo "ERROR: VirtualBox provider could not be made available"
+  return 1
+}
+
+_ensure_virtualbox
+export VAGRANT_DEFAULT_PROVIDER=virtualbox

x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile

@@ -32,8 +32,7 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provision "file", source: cachedAgentSource, destination: "~/#{cachedAgentFilename}"
-  config.vm.provision "shell", inline: "mkdir #{agentDestinationFolder}"
+  config.vm.provision "shell", inline: "mkdir -p #{agentDestinationFolder}"
   config.vm.provision "shell", inline: "tar -zxf #{cachedAgentFilename} --directory #{agentDestinationFolder} --strip-components=1 && rm -f #{cachedAgentFilename}"
-  config.vm.provision "shell", inline: "sudo apt-get update"
-  config.vm.provision "shell", inline: "sudo apt-get install unzip"
+  config.vm.provision "shell", inline: "sudo apt-get update && sudo apt-get install -y unzip"
 end

x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts

@@ -10,7 +10,7 @@ import execa from 'execa';
 import chalk from 'chalk';
 import path from 'path';
 import { userInfo } from 'os';
-import { unlink as deleteFile } from 'fs/promises';
+import { unlink as deleteFile, statfs } from 'fs/promises';
 import { dump } from './utils';
 import type { DownloadedAgentInfo } from './agent_downloads_service';
 import { BaseDataGenerator } from '../../../common/endpoint/data_generators/base_data_generator';
@@ -247,6 +247,94 @@ interface CreateVagrantVmOptions extends BaseVmCreateOptions {
   log?: ToolingLog;
 }
 
+const ensureVirtualBoxProvider = async (log: ToolingLog): Promise<void> => {
+  const isVboxKernelLoaded = async (): Promise<boolean> => {
+    try {
+      const result = await execa.command('VBoxManage --version', {
+        stdio: 'pipe',
+        all: true,
+      });
+      const combined = `${result.stdout}\n${result.stderr}`;
+      if (combined.toLowerCase().includes('kernel module is not loaded')) {
+        log.warning('VBoxManage reports kernel module not loaded');
+        return false;
+      }
+      log.info(`VirtualBox version: ${result.stdout.trim()}`);
+      return true;
+    } catch {
+      return false;
+    }
+  };
+
+  if (await isVboxKernelLoaded()) {
+    return;
+  }
+
+  const loadModules = async (): Promise<boolean> => {
+    for (const cmd of [
+      'sudo modprobe vboxdrv',
+      'sudo /sbin/vboxconfig',
+      'sudo /sbin/rcvboxdrv setup',
+    ]) {
+      try {
+        await execa.command(cmd, { stdio: 'pipe' });
+        if (await isVboxKernelLoaded()) {
+          log.info(`VirtualBox kernel module loaded via: ${cmd}`);
+          return true;
+        }
+      } catch {
+        log.debug(`Recovery command failed: ${cmd}`);
+      }
+    }
+    return false;
+  };
+
+  const upgradeToVbox71 = async (): Promise<boolean> => {
+    log.info('Upgrading to VirtualBox 7.1 (supports newer kernels)...');
+    try {
+      await execa.command('sudo apt-get install -y --no-install-recommends virtualbox-7.1', {
+        stdio: 'pipe',
+      });
+      return (await loadModules()) && (await isVboxKernelLoaded());
+    } catch {
+      log.debug('virtualbox-7.1 package not available or install failed');
+      return false;
+    }
+  };
+
+  log.warning('VirtualBox kernel module not loaded, attempting recovery...');
+
+  if (await loadModules()) {
+    return;
+  }
+
+  log.warning('Kernel module build failed, upgrading to VirtualBox 7.1...');
+
+  if (await upgradeToVbox71()) {
+    return;
+  }
+
+  const diagnostics: string[] = [];
+  for (const diagCmd of [
+    'VBoxManage --version 2>&1',
+    'lsmod | grep vbox || echo "no vbox modules"',
+    'uname -r',
+    'dpkg -l | grep -i virtualbox 2>/dev/null || echo "no vbox packages"',
+    'dkms status 2>/dev/null || echo "dkms not available"',
+  ]) {
+    try {
+      const { stdout } = await execa.command(diagCmd, { stdio: 'pipe', shell: true });
+      diagnostics.push(`${diagCmd}: ${stdout.trim()}`);
+    } catch {
+      diagnostics.push(`${diagCmd}: (failed)`);
+    }
+  }
+
+  throw new Error(
+    `VirtualBox kernel module could not be loaded.\nDiagnostics:\n${diagnostics.join('\n')}`
+  );
+};
+
 /**
  * Creates a new VM using `vagrant`
  */
@@ -263,44 +351,78 @@ const createVagrantVm = async ({
 
   const VAGRANT_CWD = path.dirname(vagrantFile);
 
-  // Destroy the VM running (if any) with the provided vagrant file before re-creating it
-  try {
-    await execa.command(`vagrant destroy -f`, {
-      env: {
-        VAGRANT_CWD,
-      },
-      // Only `pipe` STDERR to parent process
-      stdio: ['inherit', 'inherit', 'pipe'],
-    });
-    // eslint-disable-next-line no-empty
-  } catch (e) {}
-
   if (memory || cpus || disk) {
     log.warning(
       `cpu, memory and disk options ignored for creation of vm via Vagrant. These should be defined in the Vagrantfile`
     );
   }
 
+  const vagrantEnv = {
+    ...(process.env.CI ? { VAGRANT_DEFAULT_PROVIDER: 'virtualbox' } : {}),
+    VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1',
+    VAGRANT_CWD,
+    VMNAME: name,
+    CACHED_AGENT_SOURCE: agentFullFilePath,
+    CACHED_AGENT_FILENAME: agentFileName,
+    AGENT_DESTINATION_FOLDER: agentFileName.replace('.tar.gz', ''),
+  };
+
+  const MIN_DISK_GB = 10;
   try {
-    const vagrantUpResponse = (
-      await execa.command(`vagrant up`, {
-        env: {
-          VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1',
-          VAGRANT_CWD,
-          VMNAME: name,
-          CACHED_AGENT_SOURCE: agentFullFilePath,
-          CACHED_AGENT_FILENAME: agentFileName,
-          AGENT_DESTINATION_FOLDER: agentFileName.replace('.tar.gz', ''),
-        },
-        // Only `pipe` STDERR to parent process
-        stdio: ['inherit', 'inherit', 'pipe'],
-      })
-    ).stdout;
-
-    log.debug(`Vagrant up command response: `, vagrantUpResponse);
+    const stats = await statfs(VAGRANT_CWD);
+    const freeGB = (stats.bfree * stats.bsize) / 1024 / 1024 / 1024;
+    log.info(`Host disk free space: ${freeGB.toFixed(1)} GB`);
+
+    if (freeGB < MIN_DISK_GB) {
+      throw new Error(
+        `Insufficient disk space for vagrant VM: ${freeGB.toFixed(
+          1
+        )} GB free, need at least ${MIN_DISK_GB} GB`
+      );
+    }
   } catch (e) {
-    log.error(e);
-    throw e;
+    if ((e as Error).message?.includes('Insufficient disk space')) {
+      throw e;
+    }
+    log.debug(`Unable to check disk space: ${(e as Error).message}`);
+  }
+
+  if (process.env.CI) {
+    await ensureVirtualBoxProvider(log);
+  }
+
+  const MAX_ATTEMPTS = 2;
+
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    try {
+      await execa
+        .command('vagrant destroy -f', {
+          env: { VAGRANT_CWD },
+          stdio: ['inherit', 'pipe', 'pipe'],
+        })
+        .catch(() => {});
+
+      const vagrantUpResponse = (
+        await execa.command('vagrant up', {
+          env: vagrantEnv,
+          stdio: ['inherit', 'pipe', 'pipe'],
+        })
+      ).stdout;
+
+      log.debug('Vagrant up command response: ', vagrantUpResponse);
+      break;
+    } catch (e) {
+      const execError = e as execa.ExecaError;
+      log.error(
+        `vagrant up failed (attempt ${attempt}/${MAX_ATTEMPTS}):\nSTDOUT: ${execError.stdout}\nSTDERR: ${execError.stderr}`
+      );
+
+      if (attempt === MAX_ATTEMPTS) {
+        throw e;
+      }
+
+      log.info('Retrying vagrant up after cleanup...');
+    }
   }
 
   return createVagrantHostVmClient(name, undefined, log);