diff --git a/.buildkite/scripts/steps/functional/defend_workflows.sh b/.buildkite/scripts/steps/functional/defend_workflows.sh index 299a9be174477..f50057a165396 100755 --- a/.buildkite/scripts/steps/functional/defend_workflows.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows.sh @@ -7,6 +7,8 @@ source .buildkite/scripts/steps/functional/common.sh export JOB=kibana-defend-workflows-cypress export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} +source .buildkite/scripts/steps/functional/ensure_virtualbox.sh + echo "--- Defend Workflows Cypress tests" cd x-pack/solutions/security/plugins/security_solution diff --git a/.buildkite/scripts/steps/functional/defend_workflows_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_burn.sh index bbe55b02ca4cb..b7fb3891e91fe 100644 --- a/.buildkite/scripts/steps/functional/defend_workflows_burn.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows_burn.sh @@ -9,6 +9,8 @@ export JOB=kibana-defend-workflows-cypress buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false' +source .buildkite/scripts/steps/functional/ensure_virtualbox.sh + echo "--- Defend Workflows Cypress tests, burning changed specs (Chrome)" yarn --cwd x-pack/solutions/security/plugins/security_solution cypress:changed-specs-only diff --git a/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh b/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh index 21e188e476b6f..a13b763df849e 100755 --- a/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh @@ -7,6 +7,8 @@ source .buildkite/scripts/steps/functional/common.sh export JOB=kibana-defend-workflows-serverless-cypress export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} +source .buildkite/scripts/steps/functional/ensure_virtualbox.sh + echo "--- Defend Workflows Cypress tests on Serverless" cd x-pack/solutions/security/plugins/security_solution diff --git a/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh index bb6770c109300..d15c6c880e06c 100644 --- a/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh +++ b/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh @@ -9,6 +9,8 @@ export JOB=kibana-defend-workflows-serverless-cypress buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false' +source .buildkite/scripts/steps/functional/ensure_virtualbox.sh + echo "--- Defend Workflows Cypress tests, burning changed specs (Chrome)" yarn --cwd x-pack/solutions/security/plugins/security_solution cypress:dw:serverless:changed-specs-only diff --git a/.buildkite/scripts/steps/functional/ensure_virtualbox.sh b/.buildkite/scripts/steps/functional/ensure_virtualbox.sh new file mode 100755 index 0000000000000..47821f257fbf2 --- /dev/null +++ b/.buildkite/scripts/steps/functional/ensure_virtualbox.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Ensures VirtualBox is installed with working kernel modules and +# exports VAGRANT_DEFAULT_PROVIDER=virtualbox. +# Must be sourced (not executed) so the export propagates to the caller. + +echo "--- Ensure VirtualBox provider" + +_vbox_kernel_ok() { + local output + output=$(VBoxManage --version 2>&1) + if echo "$output" | grep -qi "kernel module is not loaded"; then + return 1 + fi + return 0 +} + +_ensure_vbox_modules() { + sudo modprobe vboxdrv 2>/dev/null && return 0 + sudo /sbin/vboxconfig 2>/dev/null && return 0 + sudo /sbin/rcvboxdrv setup 2>/dev/null && return 0 + return 1 +} + +_print_vbox_diagnostics() { + echo "VirtualBox diagnostics:" + echo " vagrant: $(vagrant --version 2>/dev/null || echo 'not found')" + echo " VBoxManage: $(which VBoxManage 2>/dev/null || echo 'not in PATH')" + echo " VBoxManage --version: $(VBoxManage --version 2>&1 || echo 'failed')" + echo " vbox modules: $(lsmod 2>/dev/null | grep vbox || echo 'none loaded')" + echo " vbox packages: $(dpkg -l 2>/dev/null | grep -i virtualbox | awk '{print $2, $3}' || echo 'none')" + echo " kernel: $(uname -r)" + echo " dkms status: $(dkms status 2>/dev/null || echo 'dkms not available')" +} + +_upgrade_to_vbox71() { + echo "Upgrading to VirtualBox 7.1 (supports newer kernels)..." + + # Oracle's repo is pre-configured on CI images; install 7.1 from there + if apt-cache show virtualbox-7.1 &>/dev/null; then + echo "virtualbox-7.1 available from Oracle repo, installing..." + sudo apt-get install -y --no-install-recommends virtualbox-7.1 2>&1 + return $? + fi + + # If Oracle repo isn't configured, add it + echo "Adding Oracle VirtualBox repository..." + local codename + codename=$(lsb_release -cs 2>/dev/null || echo "noble") + wget -qO- https://www.virtualbox.org/download/oracle_vbox_2016.asc | sudo gpg --dearmor --yes -o /usr/share/keyrings/oracle-virtualbox-2016.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/oracle-virtualbox-2016.gpg] https://download.virtualbox.org/virtualbox/debian ${codename} contrib" | \ + sudo tee /etc/apt/sources.list.d/virtualbox-oracle.list + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends virtualbox-7.1 2>&1 + return $? +} + +_ensure_virtualbox() { + # 1. Already working — nothing to do + if command -v VBoxManage &>/dev/null && _vbox_kernel_ok; then + echo "VirtualBox ready: $(VBoxManage --version 2>&1 | tail -1)" + return 0 + fi + + # 2. VBoxManage exists but kernel module not loaded — try loading + if command -v VBoxManage &>/dev/null; then + echo "VBoxManage found but kernel module not loaded, attempting recovery..." + if _ensure_vbox_modules && _vbox_kernel_ok; then + echo "VirtualBox kernel module recovered: $(VBoxManage --version 2>&1 | tail -1)" + return 0 + fi + + # Modules couldn't load — likely kernel too new for current VBox version. + # Upgrade to 7.1 which supports newer kernels. + echo "Kernel module build failed for current VirtualBox, upgrading to 7.1..." + if _upgrade_to_vbox71 && _ensure_vbox_modules && _vbox_kernel_ok; then + echo "VirtualBox 7.1 installed and ready: $(VBoxManage --version 2>&1 | tail -1)" + return 0 + fi + else + # 3. VBoxManage not found at all — install 7.1 from scratch + echo "VirtualBox not found, installing 7.1..." + sudo apt-get update -qq + if _upgrade_to_vbox71 && _ensure_vbox_modules && _vbox_kernel_ok; then + echo "VirtualBox 7.1 installed and ready: $(VBoxManage --version 2>&1 | tail -1)" + return 0 + fi + fi + + _print_vbox_diagnostics + echo "ERROR: VirtualBox provider could not be made available" + return 1 +} + +_ensure_virtualbox +export VAGRANT_DEFAULT_PROVIDER=virtualbox diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index b9d34d6ab3a2f..e95d092f374f8 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -32,8 +32,7 @@ Vagrant.configure("2") do |config| end config.vm.provision "file", source: cachedAgentSource, destination: "~/#{cachedAgentFilename}" - config.vm.provision "shell", inline: "mkdir #{agentDestinationFolder}" + config.vm.provision "shell", inline: "mkdir -p #{agentDestinationFolder}" config.vm.provision "shell", inline: "tar -zxf #{cachedAgentFilename} --directory #{agentDestinationFolder} --strip-components=1 && rm -f #{cachedAgentFilename}" - config.vm.provision "shell", inline: "sudo apt-get update" - config.vm.provision "shell", inline: "sudo apt-get install unzip" + config.vm.provision "shell", inline: "sudo apt-get update && sudo apt-get install -y unzip" end diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts index ec8031bbbd00b..604c4d91cea5e 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vm_services.ts @@ -10,7 +10,7 @@ import execa from 'execa'; import chalk from 'chalk'; import path from 'path'; import { userInfo } from 'os'; -import { unlink as deleteFile } from 'fs/promises'; +import { unlink as deleteFile, statfs } from 'fs/promises'; import { dump } from './utils'; import type { DownloadedAgentInfo } from './agent_downloads_service'; import { BaseDataGenerator } from '../../../common/endpoint/data_generators/base_data_generator'; @@ -242,6 +242,94 @@ interface CreateVagrantVmOptions extends BaseVmCreateOptions { log?: ToolingLog; } +const ensureVirtualBoxProvider = async (log: ToolingLog): Promise => { + const isVboxKernelLoaded = async (): Promise => { + try { + const result = await execa.command('VBoxManage --version', { + stdio: 'pipe', + all: true, + }); + const combined = `${result.stdout}\n${result.stderr}`; + if (combined.toLowerCase().includes('kernel module is not loaded')) { + log.warning('VBoxManage reports kernel module not loaded'); + return false; + } + log.info(`VirtualBox version: ${result.stdout.trim()}`); + return true; + } catch { + return false; + } + }; + + if (await isVboxKernelLoaded()) { + return; + } + + const loadModules = async (): Promise => { + for (const cmd of [ + 'sudo modprobe vboxdrv', + 'sudo /sbin/vboxconfig', + 'sudo /sbin/rcvboxdrv setup', + ]) { + try { + await execa.command(cmd, { stdio: 'pipe' }); + if (await isVboxKernelLoaded()) { + log.info(`VirtualBox kernel module loaded via: ${cmd}`); + return true; + } + } catch { + log.debug(`Recovery command failed: ${cmd}`); + } + } + return false; + }; + + const upgradeToVbox71 = async (): Promise => { + log.info('Upgrading to VirtualBox 7.1 (supports newer kernels)...'); + try { + await execa.command('sudo apt-get install -y --no-install-recommends virtualbox-7.1', { + stdio: 'pipe', + }); + return (await loadModules()) && (await isVboxKernelLoaded()); + } catch { + log.debug('virtualbox-7.1 package not available or install failed'); + return false; + } + }; + + log.warning('VirtualBox kernel module not loaded, attempting recovery...'); + + if (await loadModules()) { + return; + } + + log.warning('Kernel module build failed, upgrading to VirtualBox 7.1...'); + + if (await upgradeToVbox71()) { + return; + } + + const diagnostics: string[] = []; + for (const diagCmd of [ + 'VBoxManage --version 2>&1', + 'lsmod | grep vbox || echo "no vbox modules"', + 'uname -r', + 'dpkg -l | grep -i virtualbox 2>/dev/null || echo "no vbox packages"', + 'dkms status 2>/dev/null || echo "dkms not available"', + ]) { + try { + const { stdout } = await execa.command(diagCmd, { stdio: 'pipe', shell: true }); + diagnostics.push(`${diagCmd}: ${stdout.trim()}`); + } catch { + diagnostics.push(`${diagCmd}: (failed)`); + } + } + + throw new Error( + `VirtualBox kernel module could not be loaded.\nDiagnostics:\n${diagnostics.join('\n')}` + ); +}; + /** * Creates a new VM using `vagrant` */ @@ -258,44 +346,78 @@ const createVagrantVm = async ({ const VAGRANT_CWD = path.dirname(vagrantFile); - // Destroy the VM running (if any) with the provided vagrant file before re-creating it - try { - await execa.command(`vagrant destroy -f`, { - env: { - VAGRANT_CWD, - }, - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }); - // eslint-disable-next-line no-empty - } catch (e) {} - if (memory || cpus || disk) { log.warning( `cpu, memory and disk options ignored for creation of vm via Vagrant. These should be defined in the Vagrantfile` ); } + const vagrantEnv = { + ...(process.env.CI ? { VAGRANT_DEFAULT_PROVIDER: 'virtualbox' } : {}), + VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1', + VAGRANT_CWD, + VMNAME: name, + CACHED_AGENT_SOURCE: agentFullFilePath, + CACHED_AGENT_FILENAME: agentFileName, + AGENT_DESTINATION_FOLDER: agentFileName.replace('.tar.gz', ''), + }; + + const MIN_DISK_GB = 10; try { - const vagrantUpResponse = ( - await execa.command(`vagrant up`, { - env: { - VAGRANT_DISABLE_VBOXSYMLINKCREATE: '1', - VAGRANT_CWD, - VMNAME: name, - CACHED_AGENT_SOURCE: agentFullFilePath, - CACHED_AGENT_FILENAME: agentFileName, - AGENT_DESTINATION_FOLDER: agentFileName.replace('.tar.gz', ''), - }, - // Only `pipe` STDERR to parent process - stdio: ['inherit', 'inherit', 'pipe'], - }) - ).stdout; - - log.debug(`Vagrant up command response: `, vagrantUpResponse); + const stats = await statfs(VAGRANT_CWD); + const freeGB = (stats.bfree * stats.bsize) / 1024 / 1024 / 1024; + log.info(`Host disk free space: ${freeGB.toFixed(1)} GB`); + + if (freeGB < MIN_DISK_GB) { + throw new Error( + `Insufficient disk space for vagrant VM: ${freeGB.toFixed( + 1 + )} GB free, need at least ${MIN_DISK_GB} GB` + ); + } } catch (e) { - log.error(e); - throw e; + if ((e as Error).message?.includes('Insufficient disk space')) { + throw e; + } + log.debug(`Unable to check disk space: ${(e as Error).message}`); + } + + if (process.env.CI) { + await ensureVirtualBoxProvider(log); + } + + const MAX_ATTEMPTS = 2; + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + await execa + .command('vagrant destroy -f', { + env: { VAGRANT_CWD }, + stdio: ['inherit', 'pipe', 'pipe'], + }) + .catch(() => {}); + + const vagrantUpResponse = ( + await execa.command('vagrant up', { + env: vagrantEnv, + stdio: ['inherit', 'pipe', 'pipe'], + }) + ).stdout; + + log.debug('Vagrant up command response: ', vagrantUpResponse); + break; + } catch (e) { + const execError = e as execa.ExecaError; + log.error( + `vagrant up failed (attempt ${attempt}/${MAX_ATTEMPTS}):\nSTDOUT: ${execError.stdout}\nSTDERR: ${execError.stderr}` + ); + + if (attempt === MAX_ATTEMPTS) { + throw e; + } + + log.info('Retrying vagrant up after cleanup...'); + } } return createVagrantHostVmClient(name, undefined, log);