From 63780ab41e839cd356c36158c12e59c530d5a987 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 24 Apr 2023 11:48:40 -0400 Subject: [PATCH 1/7] Delete `getAgentDownloadUrl()` form endpoint agent runner and use common fleet service function --- .../endpoint_agent_runner/elastic_endpoint.ts | 51 +------------------ 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts index 68ff4951f77d4..b7cd8a51653ec 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts @@ -7,7 +7,6 @@ import { userInfo } from 'os'; import execa from 'execa'; -import nodeFetch from 'node-fetch'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, packagePolicyRouteService, @@ -21,29 +20,13 @@ import { fetchAgentPolicyEnrollmentKey, fetchAgentPolicyList, fetchFleetServerUrl, + getAgentDownloadUrl, waitForHostToEnroll, } from '../common/fleet_services'; import { getRuntimeServices } from './runtime'; import { type PolicyData, ProtectionModes } from '../../../common/endpoint/types'; import { dump } from './utils'; -interface ElasticArtifactSearchResponse { - manifest: { - 'last-update-time': string; - 'seconds-since-last-update': number; - }; - packages: { - [packageFileName: string]: { - architecture: string; - os: string[]; - type: string; - asc_url: string; - sha_url: string; - url: string; - }; - }; -} - export const enrollEndpointHost = async (): Promise => { let vmName; const { @@ -89,7 +72,7 @@ export const enrollEndpointHost = async (): Promise => { log.verbose(await execa('multipass', ['info', vmName])); - const agentDownloadUrl = await getAgentDownloadUrl(version); + const agentDownloadUrl = await getAgentDownloadUrl(version, falase, log); const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); @@ -155,36 +138,6 @@ export const enrollEndpointHost = async (): Promise => { return vmName; }; -const getAgentDownloadUrl = async (version: string): Promise => { - const { log } = getRuntimeServices(); - const downloadArch = - { arm64: 'arm64', x64: 'x86_64' }[process.arch] ?? `UNSUPPORTED_ARCHITECTURE_${process.arch}`; - const agentFile = `elastic-agent-${version}-linux-${downloadArch}.tar.gz`; - const artifactSearchUrl = `https://artifacts-api.elastic.co/v1/search/${version}/${agentFile}`; - - log.verbose(`Retrieving elastic agent download URL from:\n ${artifactSearchUrl}`); - - const searchResult: ElasticArtifactSearchResponse = await nodeFetch(artifactSearchUrl).then( - (response) => { - if (!response.ok) { - throw new Error( - `Failed to search elastic's artifact repository: ${response.statusText} (HTTP ${response.status})` - ); - } - - return response.json(); - } - ); - - log.verbose(searchResult); - - if (!searchResult.packages[agentFile]) { - throw new Error(`Unable to find an Agent download URL for version [${version}]`); - } - - return searchResult.packages[agentFile].url; -}; - const getOrCreateAgentPolicyId = async (): Promise => { const { kbnClient, log } = getRuntimeServices(); const username = userInfo().username.toLowerCase(); From b9b52ae2df942c4171fdaa60144311a874fa7118 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 24 Apr 2023 12:20:12 -0400 Subject: [PATCH 2/7] Added `useClosestVersionMatch` to `createAndEnrollEndpointHost()` --- .../public/management/cypress/support/data_loaders.ts | 7 ++++++- .../scripts/endpoint/common/endpoint_host_services.ts | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index ffcca01a6f1e9..4a006fdacc353 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -195,7 +195,12 @@ export const dataLoadersForRealEndpoints = ( options: Omit ): Promise => { const { kbnClient, log } = await stackServicesPromise; - return createAndEnrollEndpointHost({ ...options, log, kbnClient }).then((newHost) => { + return createAndEnrollEndpointHost({ + useClosestVersionMatch: true, + ...options, + log, + kbnClient, + }).then((newHost) => { return waitForEndpointToStreamData(kbnClient, newHost.agentId, 120000).then(() => { return newHost; }); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index 4bb03324f172e..e1792ce13fd08 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -26,6 +26,8 @@ export interface CreateAndEnrollEndpointHostOptions agentPolicyId: string; /** version of the Agent to install. Defaults to stack version */ version?: string; + /** If `version` should be exact, or if this is `true`, then the closest version will be used */ + useClosestVersionMatch?: boolean; /** The name for the host. Will also be the name of the VM */ hostname?: string; } @@ -47,6 +49,7 @@ export const createAndEnrollEndpointHost = async ({ memory, hostname, version = kibanaPackageJson.version, + useClosestVersionMatch = false, }: CreateAndEnrollEndpointHostOptions): Promise => { const [vm, agentDownloadUrl, fleetServerUrl, enrollmentToken] = await Promise.all([ createMultipassVm({ @@ -56,13 +59,15 @@ export const createAndEnrollEndpointHost = async ({ memory, }), - getAgentDownloadUrl(version, true, log), + getAgentDownloadUrl(version, useClosestVersionMatch, log), fetchFleetServerUrl(kbnClient), fetchAgentPolicyEnrollmentKey(kbnClient, agentPolicyId), ]); + log.verbose(await execa('multipass', ['info', vm.vmName])); + // Some validations before we proceed assert(agentDownloadUrl, 'Missing agent download URL'); assert(fleetServerUrl, 'Fleet server URL not set'); @@ -158,6 +163,8 @@ const enrollHostWithFleet = async ({ const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); + log.verbose(`downloading and installing agent on host`); + await execa.command( `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` ); From 6a15eeca341b9a4fe571f048ce0c9d21063dd52f Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 24 Apr 2023 12:20:29 -0400 Subject: [PATCH 3/7] Delete duplicate code from endpoint agent runner and replaced it with calls to new common services --- .../endpoint_agent_runner/elastic_endpoint.ts | 84 +++---------------- 1 file changed, 12 insertions(+), 72 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts index b7cd8a51653ec..f35901e263d58 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/elastic_endpoint.ts @@ -14,15 +14,10 @@ import { type UpdatePackagePolicy, } from '@kbn/fleet-plugin/common'; import chalk from 'chalk'; +import { createAndEnrollEndpointHost } from '../common/endpoint_host_services'; import { getEndpointPackageInfo } from '../../../common/endpoint/utils/package'; import { indexFleetEndpointPolicy } from '../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; -import { - fetchAgentPolicyEnrollmentKey, - fetchAgentPolicyList, - fetchFleetServerUrl, - getAgentDownloadUrl, - waitForHostToEnroll, -} from '../common/fleet_services'; +import { fetchAgentPolicyList } from '../common/fleet_services'; import { getRuntimeServices } from './runtime'; import { type PolicyData, ProtectionModes } from '../../../common/endpoint/types'; import { dump } from './utils'; @@ -44,81 +39,26 @@ export const enrollEndpointHost = async (): Promise => { const policyId: string = policy || (await getOrCreateAgentPolicyId()); if (!policyId) { - throw new Error(`No valid policy id provide or unable to create it`); + throw new Error(`No valid policy id provided or unable to create it`); } if (!version) { throw new Error(`No 'version' specified`); } - const [fleetServerHostUrl, enrollmentToken] = await Promise.all([ - fetchFleetServerUrl(kbnClient), - fetchAgentPolicyEnrollmentKey(kbnClient, policyId), - ]); - - if (!fleetServerHostUrl) { - throw new Error(`Fleet setting does not have a Fleet Server host defined!`); - } - - if (!enrollmentToken) { - throw new Error(`No API enrollment key found for policy id [${policyId}]`); - } - vmName = `${username}-dev-${uniqueId}`; log.info(`Creating VM named: ${vmName}`); - await execa.command(`multipass launch --name ${vmName} --disk 8G`); - - log.verbose(await execa('multipass', ['info', vmName])); - - const agentDownloadUrl = await getAgentDownloadUrl(version, falase, log); - const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); - const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); - - log.info(`Downloading and installing agent`); - log.verbose(`Agent download:\n ${agentDownloadUrl}`); - - await execa.command( - `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` - ); - await execa.command(`multipass exec ${vmName} -- tar -zxf ${agentDownloadedFile}`); - await execa.command(`multipass exec ${vmName} -- rm -f ${agentDownloadedFile}`); - - const agentInstallArguments = [ - 'exec', - - vmName, - - '--working-directory', - `/home/ubuntu/${vmDirName}`, - - '--', - - 'sudo', - - './elastic-agent', - - 'install', - - '--insecure', - - '--force', - - '--url', - fleetServerHostUrl, - - '--enrollment-token', - enrollmentToken, - ]; - - log.info(`Enrolling elastic agent with Fleet`); - log.verbose(`Command: multipass ${agentInstallArguments.join(' ')}`); - - await execa(`multipass`, agentInstallArguments); - - log.info(`Waiting for Agent to check-in with Fleet`); - await waitForHostToEnroll(kbnClient, vmName); + await createAndEnrollEndpointHost({ + kbnClient, + log, + hostname: vmName, + agentPolicyId: policyId, + version, + useClosestVersionMatch: false, + disk: '8G', + }); log.info(`VM created using Multipass. VM Name: ${vmName} From 2b8d4105510205b26b5ff940776a11847cbbe4bb Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 24 Apr 2023 17:23:43 -0400 Subject: [PATCH 4/7] Added additional methods to `SettingsStorage` --- .../endpoint/common/settings_storage.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/settings_storage.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/settings_storage.ts index d68da4bfc92b6..6fa4e762d5e9d 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/settings_storage.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/settings_storage.ts @@ -28,10 +28,8 @@ export class SettingsStorage { private dirExists: boolean = false; constructor(fileName: string, options: SettingStorageOptions = {}) { - const { - directory = join(homedir(), '.kibanaSecuritySolutionCliTools'), - defaultSettings = {} as TSettingsDef, - } = options; + const { directory = SettingsStorage.getDirectory(), defaultSettings = {} as TSettingsDef } = + options; this.options = { directory, @@ -41,7 +39,17 @@ export class SettingsStorage { this.settingsFileFullPath = join(this.options.directory, fileName); } - private async ensureExists(): Promise { + /** Returns the default path to the directory where settings are saved to. */ + public static getDirectory(): string { + return join(homedir(), '.kibanaSecuritySolutionCliTools'); + } + + /** Build a path using the root directory of where settings are saved */ + protected buildPath(path: string): string { + return join(this.options.directory, path); + } + + protected async ensureExists(): Promise { if (!this.dirExists) { await mkdir(this.options.directory, { recursive: true }); this.dirExists = true; From d94ff67ae8a3ec215227a8eb1968393064fb1324 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Mon, 24 Apr 2023 17:38:33 -0400 Subject: [PATCH 5/7] `createAndEnrollEndpointHost()` support for using agent downloads cache --- .../endpoint/common/endpoint_host_services.ts | 161 ++++++++++++++++-- 1 file changed, 148 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index e1792ce13fd08..b5512f99cf04d 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -10,6 +10,11 @@ import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import assert from 'assert'; +import { mkdir, unlink } from 'fs/promises'; +import { join } from 'path'; +import fs from 'fs'; +import { finished } from 'stream/promises'; +import nodeFetch from 'node-fetch'; import { fetchAgentPolicyEnrollmentKey, fetchFleetServerUrl, @@ -17,6 +22,7 @@ import { unEnrollFleetAgent, waitForHostToEnroll, } from './fleet_services'; +import { SettingsStorage } from './settings_storage'; export interface CreateAndEnrollEndpointHostOptions extends Pick { @@ -26,10 +32,12 @@ export interface CreateAndEnrollEndpointHostOptions agentPolicyId: string; /** version of the Agent to install. Defaults to stack version */ version?: string; - /** If `version` should be exact, or if this is `true`, then the closest version will be used */ - useClosestVersionMatch?: boolean; /** The name for the host. Will also be the name of the VM */ hostname?: string; + /** If `version` should be exact, or if this is `true`, then the closest version will be used. Defaults to `false` */ + useClosestVersionMatch?: boolean; + /** If the local cache of agent downloads should be used. Defaults to `true` */ + useCache?: boolean; } export interface CreateAndEnrollEndpointHostResponse { @@ -50,8 +58,11 @@ export const createAndEnrollEndpointHost = async ({ hostname, version = kibanaPackageJson.version, useClosestVersionMatch = false, + useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { - const [vm, agentDownloadUrl, fleetServerUrl, enrollmentToken] = await Promise.all([ + let cacheCleanupPromise = Promise.resolve(); + + const [vm, agentDownload, fleetServerUrl, enrollmentToken] = await Promise.all([ createMultipassVm({ vmName: hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`, disk, @@ -59,7 +70,25 @@ export const createAndEnrollEndpointHost = async ({ memory, }), - getAgentDownloadUrl(version, useClosestVersionMatch, log), + getAgentDownloadUrl(version, useClosestVersionMatch, log).then<{ + url: string; + cache?: DownloadedAgentInfo; + }>((url) => { + if (useCache) { + const agentDownloadClient = new AgentDownloadStorage(); + + cacheCleanupPromise = agentDownloadClient.cleanupDownloads(); + + return agentDownloadClient.downloadAndStore(url).then((cache) => { + return { + url, + cache, + }; + }); + } + + return { url }; + }), fetchFleetServerUrl(kbnClient), @@ -69,7 +98,7 @@ export const createAndEnrollEndpointHost = async ({ log.verbose(await execa('multipass', ['info', vm.vmName])); // Some validations before we proceed - assert(agentDownloadUrl, 'Missing agent download URL'); + assert(agentDownload.url, 'Missing agent download URL'); assert(fleetServerUrl, 'Fleet server URL not set'); assert(enrollmentToken, `No enrollment token for agent policy id [${agentPolicyId}]`); @@ -81,11 +110,14 @@ export const createAndEnrollEndpointHost = async ({ kbnClient, log, fleetServerUrl, - agentDownloadUrl, + agentDownloadUrl: agentDownload.url, + cachedAgentDownload: agentDownload.cache, enrollmentToken, vmName: vm.vmName, }); + await cacheCleanupPromise; + return { hostname: vm.vmName, agentId, @@ -148,6 +180,7 @@ interface EnrollHostWithFleetOptions { log: ToolingLog; vmName: string; agentDownloadUrl: string; + cachedAgentDownload?: DownloadedAgentInfo; fleetServerUrl: string; enrollmentToken: string; } @@ -158,18 +191,35 @@ const enrollHostWithFleet = async ({ vmName, fleetServerUrl, agentDownloadUrl, + cachedAgentDownload, enrollmentToken, }: EnrollHostWithFleetOptions): Promise<{ agentId: string }> => { const agentDownloadedFile = agentDownloadUrl.substring(agentDownloadUrl.lastIndexOf('/') + 1); const vmDirName = agentDownloadedFile.replace(/\.tar\.gz$/, ''); - log.verbose(`downloading and installing agent on host`); - - await execa.command( - `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` - ); - await execa.command(`multipass exec ${vmName} -- tar -zxf ${agentDownloadedFile}`); - await execa.command(`multipass exec ${vmName} -- rm -f ${agentDownloadedFile}`); + if (cachedAgentDownload) { + log.verbose( + `Installing agent on host using cached download from [${cachedAgentDownload.fullFilePath}]` + ); + + // mount local folder on VM + await execa.command( + `multipass mount ${cachedAgentDownload.directory} ${vmName}:~/_agent_downloads` + ); + await execa.command( + `multipass exec ${vmName} -- tar -zxf _agent_downloads/${cachedAgentDownload.filename}` + ); + await execa.command(`multipass unmount ${vmName}:~/_agent_downloads`); + } else { + log.verbose(`downloading and installing agent from URL [${agentDownloadUrl}]`); + + // download into VM + await execa.command( + `multipass exec ${vmName} -- curl -L ${agentDownloadUrl} -o ${agentDownloadedFile}` + ); + await execa.command(`multipass exec ${vmName} -- tar -zxf ${agentDownloadedFile}`); + await execa.command(`multipass exec ${vmName} -- rm -f ${agentDownloadedFile}`); + } const agentInstallArguments = [ 'exec', @@ -210,3 +260,88 @@ const enrollHostWithFleet = async ({ agentId: agent.id, }; }; + +interface DownloadedAgentInfo { + filename: string; + directory: string; + fullFilePath: string; +} + +class AgentDownloadStorage extends SettingsStorage { + private downloadsFolderExists = false; + private readonly downloadsDirName = 'agent_download_storage'; + + constructor() { + super('agent_download_storage_settings.json'); + } + + protected async ensureExists(): Promise { + await super.ensureExists(); + + if (!this.downloadsFolderExists) { + await mkdir(this.buildPath(this.downloadsDirName), { recursive: true }); + this.downloadsFolderExists = true; + } + } + + public getPathsForUrl(agentDownloadUrl: string): DownloadedAgentInfo { + const filename = agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); + const directory = this.buildPath(join(this.downloadsDirName)); + const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); + + return { + filename, + directory, + fullFilePath, + }; + } + + public async downloadAndStore(agentDownloadUrl: string): Promise { + // TODO: should we add "retry" attempts to file downloads? + + await this.ensureExists(); + + const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl); + + // If download is already present on disk, then just return that info. No need to re-download it + if (fs.existsSync(newDownloadInfo.fullFilePath)) { + return newDownloadInfo; + } + + try { + const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath); + const { body } = await nodeFetch(agentDownloadUrl); + + await finished(body.pipe(outputStream)); + } catch (e) { + // Try to clean up download case it failed halfway through + await unlink(newDownloadInfo.fullFilePath); + + throw e; + } + + return newDownloadInfo; + } + + public async cleanupDownloads(): Promise { + // FIXME:PT implement + // + // Read all files from downloads directory + // + // delete any `snapshot` downloads that are older than [x] amount of days old + // + // Delete any files that is older than [xx] amount of days + } +} + +export const downloadAgent = async ( + agentDownloadUrl: string +): Promise => { + const agentDownloadClient = new AgentDownloadStorage(); + const downloadedAgent = await agentDownloadClient.downloadAndStore(agentDownloadUrl); + + return { + url: agentDownloadUrl, + ...downloadedAgent, + }; +}; From 9ed10cad88cddd39216becbeda9328db90056777 Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 25 Apr 2023 09:53:47 -0400 Subject: [PATCH 6/7] added cleanup method to agent downloads storage --- .../endpoint/common/endpoint_host_services.ts | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index b5512f99cf04d..c107ed325a84a 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -10,7 +10,7 @@ import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import assert from 'assert'; -import { mkdir, unlink } from 'fs/promises'; +import { mkdir, unlink, readdir, stat } from 'fs/promises'; import { join } from 'path'; import fs from 'fs'; import { finished } from 'stream/promises'; @@ -60,8 +60,6 @@ export const createAndEnrollEndpointHost = async ({ useClosestVersionMatch = false, useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { - let cacheCleanupPromise = Promise.resolve(); - const [vm, agentDownload, fleetServerUrl, enrollmentToken] = await Promise.all([ createMultipassVm({ vmName: hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`, @@ -77,8 +75,6 @@ export const createAndEnrollEndpointHost = async ({ if (useCache) { const agentDownloadClient = new AgentDownloadStorage(); - cacheCleanupPromise = agentDownloadClient.cleanupDownloads(); - return agentDownloadClient.downloadAndStore(url).then((cache) => { return { url, @@ -116,8 +112,6 @@ export const createAndEnrollEndpointHost = async ({ vmName: vm.vmName, }); - await cacheCleanupPromise; - return { hostname: vm.vmName, agentId, @@ -270,23 +264,26 @@ interface DownloadedAgentInfo { class AgentDownloadStorage extends SettingsStorage { private downloadsFolderExists = false; private readonly downloadsDirName = 'agent_download_storage'; + private readonly downloadsDirFullPath: string; constructor() { super('agent_download_storage_settings.json'); + + this.downloadsDirFullPath = this.buildPath(this.downloadsDirName); } protected async ensureExists(): Promise { await super.ensureExists(); if (!this.downloadsFolderExists) { - await mkdir(this.buildPath(this.downloadsDirName), { recursive: true }); + await mkdir(this.downloadsDirFullPath, { recursive: true }); this.downloadsFolderExists = true; } } public getPathsForUrl(agentDownloadUrl: string): DownloadedAgentInfo { const filename = agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); - const directory = this.buildPath(join(this.downloadsDirName)); + const directory = this.downloadsDirFullPath; const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); return { @@ -323,14 +320,28 @@ class AgentDownloadStorage extends SettingsStorage { return newDownloadInfo; } - public async cleanupDownloads(): Promise { - // FIXME:PT implement - // - // Read all files from downloads directory - // - // delete any `snapshot` downloads that are older than [x] amount of days old - // - // Delete any files that is older than [xx] amount of days + public async cleanupDownloads(): Promise { + const deletedFiles: string[] = []; + const deleteFilePromises: Array> = []; + const maxAgeDate = new Date(); + + maxAgeDate.setMilliseconds(-1.728e8); // -2 days + + const allFiles = await readdir(this.downloadsDirFullPath); + + for (const fileName of allFiles) { + const filePath = join(this.downloadsDirFullPath, fileName); + const fileStats = await stat(filePath); + + if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { + deleteFilePromises.push(unlink(filePath)); + deletedFiles.push(filePath); + } + } + + await Promise.allSettled(deleteFilePromises); + + return deletedFiles; } } From 7c3c31e4461828c763c876077e0b59d58067593d Mon Sep 17 00:00:00 2001 From: Paul Tavares Date: Tue, 25 Apr 2023 10:31:14 -0400 Subject: [PATCH 7/7] Extracted agent file management to its own service --- .../common/agent_downloads_service.ts | 161 ++++++++++++++++++ .../endpoint/common/endpoint_host_services.ts | 128 ++------------ 2 files changed, 179 insertions(+), 110 deletions(-) create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts new file mode 100644 index 0000000000000..ed5d0296d61a4 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mkdir, readdir, stat, unlink } from 'fs/promises'; +import { join } from 'path'; +import fs from 'fs'; +import nodeFetch from 'node-fetch'; +import { finished } from 'stream/promises'; +import { SettingsStorage } from './settings_storage'; + +export interface DownloadedAgentInfo { + filename: string; + directory: string; + fullFilePath: string; +} + +interface AgentDownloadStorageSettings { + /** + * Last time a cleanup was ran. Date in ISO format + */ + lastCleanup: string; + + /** + * The max file age in milliseconds. Defaults to 2 days + */ + maxFileAge: number; +} + +/** + * Class for managing Agent Downloads on the local disk + * @private + */ +class AgentDownloadStorage extends SettingsStorage { + private downloadsFolderExists = false; + private readonly downloadsDirName = 'agent_download_storage'; + private readonly downloadsDirFullPath: string; + + constructor() { + super('agent_download_storage_settings.json', { + defaultSettings: { + maxFileAge: 1.728e8, // 2 days + lastCleanup: new Date().toISOString(), + }, + }); + + this.downloadsDirFullPath = this.buildPath(this.downloadsDirName); + } + + protected async ensureExists(): Promise { + await super.ensureExists(); + + if (!this.downloadsFolderExists) { + await mkdir(this.downloadsDirFullPath, { recursive: true }); + this.downloadsFolderExists = true; + } + } + + public getPathsForUrl(agentDownloadUrl: string): DownloadedAgentInfo { + const filename = agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); + const directory = this.downloadsDirFullPath; + const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); + + return { + filename, + directory, + fullFilePath, + }; + } + + public async downloadAndStore(agentDownloadUrl: string): Promise { + // TODO: should we add "retry" attempts to file downloads? + + await this.ensureExists(); + + const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl); + + // If download is already present on disk, then just return that info. No need to re-download it + if (fs.existsSync(newDownloadInfo.fullFilePath)) { + return newDownloadInfo; + } + + try { + const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath); + const { body } = await nodeFetch(agentDownloadUrl); + + await finished(body.pipe(outputStream)); + } catch (e) { + // Try to clean up download case it failed halfway through + await unlink(newDownloadInfo.fullFilePath); + + throw e; + } + + return newDownloadInfo; + } + + public async cleanupDownloads(): Promise<{ deleted: string[] }> { + const settings = await this.get(); + const maxAgeDate = new Date(); + const response: { deleted: string[] } = { deleted: [] }; + + maxAgeDate.setMilliseconds(settings.maxFileAge * -1); // `* -1` to set time back + + // If cleanup already happen within the file age, then nothing to do. Exit. + if (settings.lastCleanup > maxAgeDate.toISOString()) { + return response; + } + + await this.save({ + ...settings, + lastCleanup: new Date().toISOString(), + }); + + const deleteFilePromises: Array> = []; + const allFiles = await readdir(this.downloadsDirFullPath); + + for (const fileName of allFiles) { + const filePath = join(this.downloadsDirFullPath, fileName); + const fileStats = await stat(filePath); + + if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { + deleteFilePromises.push(unlink(filePath)); + response.deleted.push(filePath); + } + } + + await Promise.allSettled(deleteFilePromises); + + return response; + } +} + +const agentDownloadsClient = new AgentDownloadStorage(); + +/** + * Downloads the agent file provided via the input URL to a local folder on disk. If the file + * already exists on disk, then no download is actually done - the information about the cached + * version is returned instead + * @param agentDownloadUrl + */ +export const downloadAndStoreAgent = async ( + agentDownloadUrl: string +): Promise => { + const downloadedAgent = await agentDownloadsClient.downloadAndStore(agentDownloadUrl); + + return { + url: agentDownloadUrl, + ...downloadedAgent, + }; +}; + +/** + * Cleans up the old agent downloads on disk. + */ +export const cleanupDownloads = async (): ReturnType => { + return agentDownloadsClient.cleanupDownloads(); +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts index c107ed325a84a..5b249ee238436 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts @@ -10,11 +10,8 @@ import type { KbnClient } from '@kbn/test'; import type { ToolingLog } from '@kbn/tooling-log'; import execa from 'execa'; import assert from 'assert'; -import { mkdir, unlink, readdir, stat } from 'fs/promises'; -import { join } from 'path'; -import fs from 'fs'; -import { finished } from 'stream/promises'; -import nodeFetch from 'node-fetch'; +import type { DownloadedAgentInfo } from './agent_downloads_service'; +import { cleanupDownloads, downloadAndStoreAgent } from './agent_downloads_service'; import { fetchAgentPolicyEnrollmentKey, fetchFleetServerUrl, @@ -22,7 +19,6 @@ import { unEnrollFleetAgent, waitForHostToEnroll, } from './fleet_services'; -import { SettingsStorage } from './settings_storage'; export interface CreateAndEnrollEndpointHostOptions extends Pick { @@ -60,6 +56,10 @@ export const createAndEnrollEndpointHost = async ({ useClosestVersionMatch = false, useCache = true, }: CreateAndEnrollEndpointHostOptions): Promise => { + let cacheCleanupPromise: ReturnType = Promise.resolve({ + deleted: [], + }); + const [vm, agentDownload, fleetServerUrl, enrollmentToken] = await Promise.all([ createMultipassVm({ vmName: hostname ?? `test-host-${Math.random().toString().substring(2, 6)}`, @@ -73,9 +73,9 @@ export const createAndEnrollEndpointHost = async ({ cache?: DownloadedAgentInfo; }>((url) => { if (useCache) { - const agentDownloadClient = new AgentDownloadStorage(); + cacheCleanupPromise = cleanupDownloads(); - return agentDownloadClient.downloadAndStore(url).then((cache) => { + return downloadAndStoreAgent(url).then((cache) => { return { url, cache, @@ -112,6 +112,16 @@ export const createAndEnrollEndpointHost = async ({ vmName: vm.vmName, }); + await cacheCleanupPromise.then((results) => { + if (results.deleted.length > 0) { + log.verbose(`Agent Downloads cache directory was cleaned up and the following ${ + results.deleted.length + } were deleted: +${results.deleted.join('\n')} +`); + } + }); + return { hostname: vm.vmName, agentId, @@ -254,105 +264,3 @@ const enrollHostWithFleet = async ({ agentId: agent.id, }; }; - -interface DownloadedAgentInfo { - filename: string; - directory: string; - fullFilePath: string; -} - -class AgentDownloadStorage extends SettingsStorage { - private downloadsFolderExists = false; - private readonly downloadsDirName = 'agent_download_storage'; - private readonly downloadsDirFullPath: string; - - constructor() { - super('agent_download_storage_settings.json'); - - this.downloadsDirFullPath = this.buildPath(this.downloadsDirName); - } - - protected async ensureExists(): Promise { - await super.ensureExists(); - - if (!this.downloadsFolderExists) { - await mkdir(this.downloadsDirFullPath, { recursive: true }); - this.downloadsFolderExists = true; - } - } - - public getPathsForUrl(agentDownloadUrl: string): DownloadedAgentInfo { - const filename = agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#'); - const directory = this.downloadsDirFullPath; - const fullFilePath = this.buildPath(join(this.downloadsDirName, filename)); - - return { - filename, - directory, - fullFilePath, - }; - } - - public async downloadAndStore(agentDownloadUrl: string): Promise { - // TODO: should we add "retry" attempts to file downloads? - - await this.ensureExists(); - - const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl); - - // If download is already present on disk, then just return that info. No need to re-download it - if (fs.existsSync(newDownloadInfo.fullFilePath)) { - return newDownloadInfo; - } - - try { - const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath); - const { body } = await nodeFetch(agentDownloadUrl); - - await finished(body.pipe(outputStream)); - } catch (e) { - // Try to clean up download case it failed halfway through - await unlink(newDownloadInfo.fullFilePath); - - throw e; - } - - return newDownloadInfo; - } - - public async cleanupDownloads(): Promise { - const deletedFiles: string[] = []; - const deleteFilePromises: Array> = []; - const maxAgeDate = new Date(); - - maxAgeDate.setMilliseconds(-1.728e8); // -2 days - - const allFiles = await readdir(this.downloadsDirFullPath); - - for (const fileName of allFiles) { - const filePath = join(this.downloadsDirFullPath, fileName); - const fileStats = await stat(filePath); - - if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { - deleteFilePromises.push(unlink(filePath)); - deletedFiles.push(filePath); - } - } - - await Promise.allSettled(deleteFilePromises); - - return deletedFiles; - } -} - -export const downloadAgent = async ( - agentDownloadUrl: string -): Promise => { - const agentDownloadClient = new AgentDownloadStorage(); - const downloadedAgent = await agentDownloadClient.downloadAndStore(agentDownloadUrl); - - return { - url: agentDownloadUrl, - ...downloadedAgent, - }; -};