diff --git a/x-pack/solutions/security/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts b/x-pack/solutions/security/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts index c4110ad2b3e0e..6f2e6c6f3e083 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/management/cypress/support/create_and_enroll_endpoint_host_ci.ts @@ -91,8 +91,12 @@ export const createAndEnrollEndpointHostCI = async ({ log.warning( `There is no agent installer for ${agentFileName} present on disk, trying to download it now.` ); - const { url: agentUrl } = await getAgentDownloadUrl(agentVersion, useClosestVersionMatch, log); - agentDownload = await downloadAndStoreAgent(agentUrl, agentFileName); + const { url: agentUrl, shaUrl } = await getAgentDownloadUrl( + agentVersion, + useClosestVersionMatch, + log + ); + agentDownload = await downloadAndStoreAgent(agentUrl, agentFileName, shaUrl); } const hostVm = await createVm({ diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts index a39ad186e62b6..c2c9e7cc1626e 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.test.ts @@ -29,10 +29,11 @@ describe('agentDownloaderRunner', () => { const version = '8.15.0'; let closestMatch = false; const url = 'http://example.com/agent.tar.gz'; + const shaUrl = 'http://example.com/agent.tar.gz.sha512'; const fileName = 'elastic-agent-8.15.0.tar.gz'; it('downloads and stores the specified version', async () => { - (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url, shaUrl }); (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); @@ -43,12 +44,12 @@ describe('agentDownloaderRunner', () => { expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log); expect(getAgentFileName).toHaveBeenCalledWith(version); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName, shaUrl); expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); }); it('logs an error if the download fails', async () => { - (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url, shaUrl }); (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); (downloadAndStoreAgent as jest.Mock).mockRejectedValue(new Error('Download failed')); @@ -59,7 +60,7 @@ describe('agentDownloaderRunner', () => { expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log); expect(getAgentFileName).toHaveBeenCalledWith(version); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName, shaUrl); expect(log.error).toHaveBeenCalledWith( 'Failed to download or store version 8.15.0: Download failed' ); @@ -70,8 +71,8 @@ describe('agentDownloaderRunner', () => { const fallbackFileName = 'elastic-agent-8.15.0.tar.gz'; (getAgentDownloadUrl as jest.Mock) - .mockResolvedValueOnce({ url }) - .mockResolvedValueOnce({ url }); + .mockResolvedValueOnce({ url, shaUrl }) + .mockResolvedValueOnce({ url, shaUrl }); (getAgentFileName as jest.Mock) .mockReturnValueOnce('elastic-agent-8.15.1') .mockReturnValueOnce('elastic-agent-8.15.0'); @@ -88,8 +89,8 @@ describe('agentDownloaderRunner', () => { expect(getAgentDownloadUrl).toHaveBeenCalledWith(fallbackVersion, closestMatch, log); expect(getAgentFileName).toHaveBeenCalledWith('8.15.1'); expect(getAgentFileName).toHaveBeenCalledWith(fallbackVersion); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz'); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fallbackFileName); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz', shaUrl); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fallbackFileName, shaUrl); expect(log.error).toHaveBeenCalledWith( 'Failed to download or store version 8.15.1: Download failed' ); @@ -97,7 +98,7 @@ describe('agentDownloaderRunner', () => { }); it('logs an error if all downloads fail', async () => { - (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url, shaUrl }); (getAgentFileName as jest.Mock) .mockReturnValueOnce('elastic-agent-8.15.1') .mockReturnValueOnce('elastic-agent-8.15.0'); @@ -114,8 +115,8 @@ describe('agentDownloaderRunner', () => { expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.0', closestMatch, log); expect(getAgentFileName).toHaveBeenCalledWith('8.15.1'); expect(getAgentFileName).toHaveBeenCalledWith('8.15.0'); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz'); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.0.tar.gz'); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz', shaUrl); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.0.tar.gz', shaUrl); expect(log.error).toHaveBeenCalledWith( 'Failed to download or store version 8.15.1: Download failed' ); @@ -125,7 +126,7 @@ describe('agentDownloaderRunner', () => { }); it('does not attempt fallback when patch version is 0', async () => { - (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url, shaUrl }); (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); @@ -136,7 +137,7 @@ describe('agentDownloaderRunner', () => { expect(getAgentDownloadUrl).toHaveBeenCalledTimes(1); // Only one call for 8.15.0 expect(getAgentFileName).toHaveBeenCalledTimes(1); - expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName); + expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName, shaUrl); expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0'); }); @@ -154,7 +155,7 @@ describe('agentDownloaderRunner', () => { it('passes the closestMatch flag correctly', async () => { closestMatch = true; - (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url }); + (getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url, shaUrl }); (getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0'); (downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined); @@ -179,8 +180,8 @@ describe('agentDownloaderRunner', () => { const primaryVersion = '8.15.1'; (getAgentDownloadUrl as jest.Mock) - .mockResolvedValueOnce({ url }) - .mockResolvedValueOnce({ url }); + .mockResolvedValueOnce({ url, shaUrl }) + .mockResolvedValueOnce({ url, shaUrl }); (getAgentFileName as jest.Mock) .mockReturnValueOnce('elastic-agent-8.15.1') diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts index a3e250a464b9c..77bffda7dfbdf 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/agent_downloader_cli/agent_downloader.ts @@ -48,10 +48,10 @@ const downloadAndStoreElasticAgent = async ( // Download all the versions in the list for (const versionToDownload of versionsToDownload) { try { - const { url } = await getAgentDownloadUrl(versionToDownload, closestMatch, log); + const { url, shaUrl } = await getAgentDownloadUrl(versionToDownload, closestMatch, log); const fileName = `${getAgentFileName(versionToDownload)}.tar.gz`; - await downloadAndStoreAgent(url, fileName); + await downloadAndStoreAgent(url, fileName, shaUrl); log.info(`Successfully downloaded and stored version ${versionToDownload}`); } catch (error) { log.error(`Failed to download or store version ${versionToDownload}: ${error.message}`); diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts index 14031a72c8e5c..130c50c0c5416 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.test.ts @@ -4,91 +4,401 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -// Adjust path if needed -import { downloadAndStoreAgent, isAgentDownloadFromDiskAvailable } from './agent_downloads_service'; +import { + downloadAndStoreAgent, + isAgentDownloadFromDiskAvailable, + fetchExpectedHash, + cleanupDownloads, +} from './agent_downloads_service'; import fs from 'fs'; -import { finished } from 'stream/promises'; +import { readFile, unlink, writeFile } from 'fs/promises'; const mockedFetch = jest.spyOn(global, 'fetch'); +const mockDigest = jest.fn(); jest.mock('fs'); +jest.mock('fs/promises', () => ({ + mkdir: jest.fn().mockResolvedValue(undefined), + readdir: jest.fn().mockResolvedValue([]), + stat: jest.fn(), + unlink: jest.fn().mockResolvedValue(undefined), + writeFile: jest.fn().mockResolvedValue(undefined), + readFile: jest.fn(), +})); jest.mock('stream/promises', () => ({ - finished: jest.fn(), + finished: jest.fn().mockResolvedValue(undefined), +})); +jest.mock('stream', () => { + const actual = jest.requireActual('stream'); + return { + ...actual, + Readable: { + ...actual.Readable, + fromWeb: jest.fn().mockReturnValue({ + pipe: jest.fn().mockReturnThis(), + }), + }, + }; +}); +jest.mock('crypto', () => ({ + createHash: jest.fn(() => ({ + digest: mockDigest, + update: jest.fn(), + })), })); jest.mock('../../../common/endpoint/data_loaders/utils', () => ({ createToolingLogger: jest.fn(() => ({ debug: jest.fn(), info: jest.fn(), + warning: jest.fn(), error: jest.fn(), })), })); +const settingsJson = JSON.stringify({ + lastCleanup: new Date(0).toISOString(), + maxFileAge: 1.728e8, +}); + +const url = 'http://example.com/agent.tar.gz'; +const shaUrl = 'http://example.com/agent.tar.gz.sha512'; +const fileName = 'elastic-agent-7.10.0.tar.gz'; +const expectedHash = 'abc123def456'; + +const mockWriteStream = { + on: jest.fn().mockReturnThis(), + once: jest.fn().mockReturnThis(), + emit: jest.fn().mockReturnThis(), + end: jest.fn(), + write: jest.fn().mockReturnValue(true), + removeListener: jest.fn().mockReturnThis(), + removeAllListeners: jest.fn().mockReturnThis(), + writable: true, +}; + +/** + * Helper to mock fs.existsSync based on file path patterns. + * Returns a function that checks the path against the provided map. + */ +const mockExistsSync = (pathResults: Record) => { + (fs.existsSync as unknown as jest.Mock).mockImplementation((path: string) => { + for (const [pattern, result] of Object.entries(pathResults)) { + if (path.includes(pattern)) return result; + } + return false; + }); +}; + +/** + * Helper to mock readFile based on path patterns. + */ +const mockReadFile = (pathResults: Record) => { + (readFile as unknown as jest.Mock).mockImplementation((path: string) => { + for (const [pattern, result] of Object.entries(pathResults)) { + if (path.includes(pattern)) return Promise.resolve(result); + } + return Promise.resolve(settingsJson); + }); +}; + describe('AgentDownloadStorage', () => { - const url = 'http://example.com/agent.tar.gz'; - const fileName = 'elastic-agent-7.10.0.tar.gz'; beforeEach(() => { - jest.clearAllMocks(); // Ensure no previous test state affects the current one + jest.clearAllMocks(); + mockDigest.mockReturnValue(expectedHash); + // Mock createReadStream used by computeFileHash — supports async iteration + const mockReadStream = { + async *[Symbol.asyncIterator]() { + yield Buffer.from('mock-data'); + }, + }; + (fs.createReadStream as unknown as jest.Mock).mockReturnValue(mockReadStream); }); - it('downloads and stores the agent if not cached', async () => { - (fs.existsSync as unknown as jest.Mock).mockReturnValue(false); - // Create a more complete mock WriteStream that supports pipe operations - const mockWriteStream = { - on: jest.fn().mockReturnThis(), - once: jest.fn().mockReturnThis(), - emit: jest.fn().mockReturnThis(), - end: jest.fn(), - write: jest.fn().mockReturnValue(true), - removeListener: jest.fn().mockReturnThis(), - removeAllListeners: jest.fn().mockReturnThis(), - writable: true, - }; - (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); - // Create a mock Response with all required properties - // Using a plain object that satisfies the Response interface used by the code - const mockBody = new ReadableStream(); - const mockResponse = { - ok: true, - status: 200, - statusText: 'OK', - body: mockBody, - } as unknown as Response; - mockedFetch.mockResolvedValue(mockResponse); - (finished as unknown as jest.Mock).mockResolvedValue(undefined); - - const result = await downloadAndStoreAgent(url, fileName); - - expect(result).toEqual({ - url, - filename: fileName, - directory: expect.any(String), - fullFilePath: expect.stringContaining(fileName), // Dynamically match the file path + describe('downloadAndStoreAgent', () => { + it('downloads and stores the agent if not cached', async () => { + mockExistsSync({ [fileName]: false }); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + mockReadFile({}); + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + }); + + it('reuses cached agent when sidecar hash matches', async () => { + mockExistsSync({ [fileName]: true, '.sha512': true }); + mockReadFile({ '.sha512': expectedHash }); + mockDigest.mockReturnValue(expectedHash); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + expect(mockedFetch).not.toHaveBeenCalled(); + }); + + it('re-downloads when cached file hash does not match sidecar', async () => { + // First call: cache check finds file + sidecar. After delete, they're gone. + let deleted = false; + (fs.existsSync as unknown as jest.Mock).mockImplementation((path: string) => { + if (deleted) return false; + if (path.includes(fileName) || path.includes('.sha512')) return true; + return false; + }); + (unlink as unknown as jest.Mock).mockImplementation(() => { + deleted = true; + return Promise.resolve(); + }); + + mockReadFile({ '.sha512': expectedHash }); + mockDigest.mockReturnValue('wrong_hash'); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + expect(unlink).toHaveBeenCalled(); + }); + + it('re-downloads when sidecar file is missing', async () => { + let deleted = false; + (fs.existsSync as unknown as jest.Mock).mockImplementation((path: string) => { + if (deleted) return false; + if (path.includes('.sha512')) return false; + if (path.includes(fileName)) return true; + return false; + }); + (unlink as unknown as jest.Mock).mockImplementation(() => { + deleted = true; + return Promise.resolve(); + }); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + mockReadFile({}); + + mockedFetch.mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + const result = await downloadAndStoreAgent(url, fileName); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + expect(mockedFetch).toHaveBeenCalledWith(url); + }); + + it('validates hash after fresh download with shaUrl', async () => { + mockExistsSync({ [fileName]: false }); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + mockDigest.mockReturnValue(expectedHash); + mockReadFile({}); + + mockedFetch + .mockResolvedValueOnce({ + ok: true, + text: jest.fn().mockResolvedValue(`${expectedHash} agent.tar.gz`), + } as unknown as Response) + .mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + const result = await downloadAndStoreAgent(url, fileName, shaUrl); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + expect(mockedFetch).toHaveBeenCalledWith(shaUrl); + expect(writeFile).toHaveBeenCalledWith( + expect.stringContaining('.sha512'), + expectedHash, + 'utf-8' + ); + }); + + it('throws after all retry attempts fail hash validation', async () => { + mockExistsSync({ [fileName]: false }); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + mockDigest.mockReturnValue('wrong_hash_every_time'); + mockReadFile({}); + + // sha_url fetch succeeds, all 3 download attempts produce wrong hash + mockedFetch + .mockResolvedValueOnce({ + ok: true, + text: jest.fn().mockResolvedValue(`${expectedHash} agent.tar.gz`), + } as unknown as Response) + .mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + await expect(downloadAndStoreAgent(url, fileName, shaUrl)).rejects.toThrow( + /Integrity check failed/ + ); + }); + + it('proceeds without validation when sha_url fetch fails but still writes local hash sidecar', async () => { + mockExistsSync({ [fileName]: false }); + (fs.createWriteStream as unknown as jest.Mock).mockReturnValue(mockWriteStream); + mockReadFile({}); + mockDigest.mockReturnValue(expectedHash); + + mockedFetch + .mockResolvedValueOnce({ + ok: false, + status: 404, + statusText: 'Not Found', + } as unknown as Response) + .mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: 'OK', + body: {}, + } as unknown as Response); + + const result = await downloadAndStoreAgent(url, fileName, shaUrl); + + expect(result).toEqual({ + url, + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + // Sidecar should be written with locally computed hash even when remote hash is unavailable + expect(writeFile).toHaveBeenCalledWith( + expect.stringContaining('.sha512'), + expectedHash, + 'utf-8' + ); }); }); - it('reuses cached agent if available', async () => { - (fs.existsSync as unknown as jest.Mock).mockReturnValue(true); + describe('isAgentDownloadFromDiskAvailable', () => { + it('returns info when file and sidecar exist', () => { + mockExistsSync({ [fileName]: true, '.sha512': true }); - const result = await downloadAndStoreAgent(url, fileName); + const result = isAgentDownloadFromDiskAvailable(fileName); - expect(result).toEqual({ - url, - filename: fileName, - directory: expect.any(String), - fullFilePath: expect.stringContaining(fileName), // Dynamically match the path + expect(result).toEqual({ + filename: fileName, + directory: expect.any(String), + fullFilePath: expect.stringContaining(fileName), + }); + }); + + it('returns undefined when file exists but sidecar is missing', () => { + (fs.existsSync as unknown as jest.Mock).mockImplementation((path: string) => { + if (path.includes('.sha512')) return false; + if (path.includes(fileName)) return true; + return false; + }); + + const result = isAgentDownloadFromDiskAvailable(fileName); + + expect(result).toBeUndefined(); + }); + + it('returns undefined when file does not exist', () => { + mockExistsSync({ [fileName]: false }); + + const result = isAgentDownloadFromDiskAvailable(fileName); + + expect(result).toBeUndefined(); }); }); - it('checks if agent download is available from disk', () => { - (fs.existsSync as unknown as jest.Mock).mockReturnValue(true); + describe('cleanupDownloads', () => { + it('deletes sidecar file alongside expired tarball', async () => { + const { readdir, stat } = jest.requireMock('fs/promises'); + const oldDate = new Date(Date.now() - 1.728e8 - 1000); // older than maxFileAge + + // Settings with old lastCleanup to trigger cleanup + const oldSettings = JSON.stringify({ + lastCleanup: new Date(0).toISOString(), + maxFileAge: 1.728e8, + }); + (readFile as unknown as jest.Mock).mockResolvedValue(oldSettings); + readdir.mockResolvedValue([fileName, `${fileName}.sha512`]); + stat.mockResolvedValue({ isFile: () => true, birthtime: oldDate }); + (unlink as unknown as jest.Mock).mockResolvedValue(undefined); + (writeFile as unknown as jest.Mock).mockResolvedValue(undefined); + + const result = await cleanupDownloads(); + + // Should have deleted the tarball + expect(result.deleted.length).toBe(1); + expect(result.deleted[0]).toContain(fileName); + // Should have also attempted to delete the sidecar + const unlinkCalls = (unlink as unknown as jest.Mock).mock.calls.map( + (call: string[]) => call[0] + ); + expect(unlinkCalls.some((path: string) => path.endsWith('.sha512'))).toBe(true); + }); + }); + + describe('fetchExpectedHash', () => { + it('parses hash from sha_url response', async () => { + mockedFetch.mockResolvedValue({ + ok: true, + text: jest + .fn() + .mockResolvedValue(`${expectedHash} elastic-agent-8.15.0-linux-x86_64.tar.gz`), + } as unknown as Response); + + const hash = await fetchExpectedHash(shaUrl); + expect(hash).toBe(expectedHash); + }); - const result = isAgentDownloadFromDiskAvailable(fileName); + it('throws on non-ok response', async () => { + mockedFetch.mockResolvedValue({ + ok: false, + status: 404, + statusText: 'Not Found', + } as unknown as Response); - expect(result).toEqual({ - filename: fileName, - directory: expect.any(String), - fullFilePath: expect.stringContaining(fileName), // Dynamically match the path + await expect(fetchExpectedHash(shaUrl)).rejects.toThrow('Failed to fetch hash'); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts index e86756a180d11..33d6050d69f58 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/agent_downloads_service.ts @@ -6,16 +6,46 @@ */ import pRetry from 'p-retry'; -import { mkdir, readdir, stat, unlink } from 'fs/promises'; +import { mkdir, readdir, stat, unlink, writeFile, readFile } from 'fs/promises'; import { join } from 'path'; import fs from 'fs'; +import { createHash } from 'crypto'; import { finished } from 'stream/promises'; import { Readable } from 'stream'; import type { ReadableStream as WebReadableStream } from 'stream/web'; +import type { ReadStream } from 'fs'; import { handleProcessInterruptions } from './nodejs_utils'; import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils'; import { SettingsStorage } from './settings_storage'; +/** + * Fetches the expected SHA512 hash from the artifacts API sha_url endpoint. + * The response is a text file in the format: " " + */ +export const fetchExpectedHash = async (shaUrl: string): Promise => { + const response = await fetch(shaUrl); + if (!response.ok) { + throw new Error( + `Failed to fetch hash from ${shaUrl}: ${response.status} ${response.statusText}` + ); + } + const text = await response.text(); + // Format is " " — extract just the hash + return text.trim().split(/\s+/)[0]; +}; + +/** + * Computes the SHA512 hash of a local file using streaming. + */ +export const computeFileHash = async (filePath: string): Promise => { + const hash = createHash('sha512'); + const stream: ReadStream = fs.createReadStream(filePath); + for await (const chunk of stream) { + hash.update(chunk); + } + return hash.digest('hex'); +}; + export interface DownloadedAgentInfo { filename: string; /** The local directory where downloads are stored */ @@ -88,22 +118,64 @@ class AgentDownloadStorage extends SettingsStorage /** * Downloads the agent and stores it locally. Reuses existing downloads if available. + * When shaUrl is provided, validates file integrity using SHA512 hash. */ public async downloadAndStore( agentDownloadUrl: string, - agentFileName?: string + agentFileName?: string, + shaUrl?: string ): Promise { this.log.debug(`Starting download: ${agentDownloadUrl}`); await this.ensureExists(); const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl, agentFileName); + const sidecarPath = `${newDownloadInfo.fullFilePath}.sha512`; - // Return cached version if the file already exists + // Check cached version with integrity validation if (fs.existsSync(newDownloadInfo.fullFilePath)) { - this.log.debug(`Download already cached at [${newDownloadInfo.fullFilePath}]`); - return newDownloadInfo; + if (fs.existsSync(sidecarPath)) { + try { + const expectedHash = (await readFile(sidecarPath, 'utf-8')).trim(); + const actualHash = await computeFileHash(newDownloadInfo.fullFilePath); + if (expectedHash === actualHash) { + this.log.debug( + `Download already cached and verified at [${newDownloadInfo.fullFilePath}]` + ); + return newDownloadInfo; + } + this.log.error( + `Cached file integrity check failed for [${newDownloadInfo.fullFilePath}] — expected ${expectedHash}, got ${actualHash}. Re-downloading.` + ); + } catch (error) { + this.log.error( + `Error validating cached file: ${(error as Error).message}. Re-downloading.` + ); + } + } else { + this.log.info( + `Cached file [${newDownloadInfo.fullFilePath}] has no sidecar hash file. Re-downloading to ensure integrity.` + ); + } + // Delete corrupt/unverifiable cached file and sidecar + await this.deleteCachedFileAndSidecar(newDownloadInfo.fullFilePath, sidecarPath); + } + + // Fetch expected hash (best-effort) + let expectedHash: string | undefined; + if (shaUrl) { + try { + expectedHash = await fetchExpectedHash(shaUrl); + } catch (error) { + this.log.warning( + `Failed to fetch SHA512 hash from [${shaUrl}]: ${ + (error as Error).message + }. Proceeding without integrity validation.` + ); + } } + let downloadedFileHash: string | undefined; + try { await pRetry( async (attempt) => { @@ -122,13 +194,29 @@ class AgentDownloadStorage extends SettingsStorage const nodeStream = Readable.fromWeb(response.body as WebReadableStream); await finished(nodeStream.pipe(outputStream)); } catch (error) { - this.log.error(`Error during download attempt ${attempt}: ${error.message}`); - // Ensure any errors here propagate and trigger retry + this.log.error( + `Error during download attempt ${attempt}: ${(error as Error).message}` + ); throw error; } }, - () => fs.unlinkSync(newDownloadInfo.fullFilePath) // Clean up on interruption + () => fs.unlinkSync(newDownloadInfo.fullFilePath) ); + + // Validate hash after download + const actualHash = await computeFileHash(newDownloadInfo.fullFilePath); + if (expectedHash) { + if (expectedHash !== actualHash) { + throw new Error( + `Integrity check failed: expected SHA512 ${expectedHash}, got ${actualHash}` + ); + } + this.log.info(`SHA512 integrity check passed for [${newDownloadInfo.fullFilePath}]`); + } + + // Store computed hash for sidecar (use remote hash if available, otherwise local) + downloadedFileHash = expectedHash || actualHash; + this.log.info(`Successfully downloaded agent to [${newDownloadInfo.fullFilePath}]`); }, { @@ -141,13 +229,31 @@ class AgentDownloadStorage extends SettingsStorage } ); } catch (error) { - throw new Error(`Download failed after multiple attempts: ${error.message}`); + throw new Error(`Download failed after multiple attempts: ${(error as Error).message}`); + } + + // Write sidecar hash file (always write — use remote hash if available, otherwise local) + if (downloadedFileHash) { + await writeFile(sidecarPath, downloadedFileHash, 'utf-8'); } await this.cleanupDownloads(); return newDownloadInfo; } + private async deleteCachedFileAndSidecar(filePath: string, sidecarPath: string): Promise { + try { + await unlink(filePath); + } catch { + // Ignore if already deleted + } + try { + await unlink(sidecarPath); + } catch { + // Ignore if sidecar doesn't exist + } + } + public async cleanupDownloads(): Promise<{ deleted: string[] }> { this.log.debug('Performing cleanup of cached Agent downloads'); @@ -168,14 +274,21 @@ class AgentDownloadStorage extends SettingsStorage try { const allFiles = await readdir(this.downloadsDirFullPath); const deleteFilePromises = allFiles.map(async (fileName) => { + // Skip sidecar files — they'll be cleaned up with their parent tarball + if (fileName.endsWith('.sha512')) { + return; + } const filePath = join(this.downloadsDirFullPath, fileName); const fileStats = await stat(filePath); if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) { try { await unlink(filePath); response.deleted.push(filePath); + // Also delete the sidecar hash file if it exists + const sidecarPath = `${filePath}.sha512`; + await unlink(sidecarPath).catch(() => {}); } catch (err) { - this.log.error(`Failed to delete file [${filePath}]: ${err.message}`); + this.log.error(`Failed to delete file [${filePath}]: ${(err as Error).message}`); } } }); @@ -184,22 +297,28 @@ class AgentDownloadStorage extends SettingsStorage this.log.debug(`Deleted ${response.deleted.length} file(s)`); return response; } catch (err) { - this.log.error(`Error during cleanup: ${err.message}`); + this.log.error(`Error during cleanup: ${(err as Error).message}`); return response; } } /** * Checks if a specific agent download is available locally. + * Returns undefined if the file exists but has no sidecar hash file (unverifiable). */ public isAgentDownloadFromDiskAvailable(filename: string): DownloadedAgentInfo | undefined { const filePath = join(this.downloadsDirFullPath, filename); if (fs.existsSync(filePath)) { + const sidecarPath = `${filePath}.sha512`; + if (!fs.existsSync(sidecarPath)) { + this.log.info( + `Cached file [${filePath}] has no sidecar hash file — treating as unavailable to force re-download with validation.` + ); + return undefined; + } return { filename, - /** The local directory where downloads are stored */ directory: this.downloadsDirFullPath, - /** The full local file path and name */ fullFilePath: filePath, }; } @@ -215,17 +334,21 @@ export interface DownloadAndStoreAgentResponse extends DownloadedAgentInfo { /** * Downloads the agent file provided via the input URL to a local folder on disk. If the file * already exists on disk, then no download is actually done - the information about the cached - * version is returned instead + * version is returned instead. + * When shaUrl is provided, validates file integrity using SHA512 hash. * @param agentDownloadUrl * @param agentFileName + * @param shaUrl */ export const downloadAndStoreAgent = async ( agentDownloadUrl: string, - agentFileName?: string + agentFileName?: string, + shaUrl?: string ): Promise => { const downloadedAgent = await agentDownloadsClient.downloadAndStore( agentDownloadUrl, - agentFileName + agentFileName, + shaUrl ); return { diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_services.ts b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_services.ts index 4165f38e535e3..441fb1ffb0ad2 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_services.ts +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/fleet_services.ts @@ -590,6 +590,8 @@ interface ElasticArtifactSearchResponse { interface GetAgentDownloadUrlResponse { url: string; + /** The URL to the SHA512 hash file for integrity validation */ + shaUrl: string; /** The file name (ex. the `*.tar.gz` file) */ fileName: string; /** The directory name that the download archive will be extracted to (same as `fileName` but no file extensions) */ @@ -639,6 +641,7 @@ export const getAgentDownloadUrl = async ( return { url: searchResult.packages[agentFile].url, + shaUrl: searchResult.packages[agentFile].sha_url, fileName: agentFile, dirName: fileNameWithoutExtension, }; @@ -853,7 +856,7 @@ export const enrollHostVmWithFleet = async ({ const agentUrlInfo = await getAgentDownloadUrl(agentVersion, closestVersionMatch, log); const agentDownload: DownloadAndStoreAgentResponse = useAgentCache - ? await downloadAndStoreAgent(agentUrlInfo.url) + ? await downloadAndStoreAgent(agentUrlInfo.url, undefined, agentUrlInfo.shaUrl) : { url: agentUrlInfo.url, directory: '', filename: agentUrlInfo.fileName, fullFilePath: '' }; log.info(`Installing Elastic Agent`); diff --git a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile index e95d092f374f8..952bfc0fa3dcb 100644 --- a/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile +++ b/x-pack/solutions/security/plugins/security_solution/scripts/endpoint/common/vagrant/Vagrantfile @@ -33,6 +33,7 @@ Vagrant.configure("2") do |config| config.vm.provision "file", source: cachedAgentSource, destination: "~/#{cachedAgentFilename}" config.vm.provision "shell", inline: "mkdir -p #{agentDestinationFolder}" - config.vm.provision "shell", inline: "tar -zxf #{cachedAgentFilename} --directory #{agentDestinationFolder} --strip-components=1 && rm -f #{cachedAgentFilename}" + config.vm.provision "shell", inline: "gzip -t /home/vagrant/#{cachedAgentFilename} || (echo 'Agent tarball integrity check failed' && exit 1)" + config.vm.provision "shell", inline: "tar -zxf /home/vagrant/#{cachedAgentFilename} --directory #{agentDestinationFolder} --strip-components=1 && rm -f /home/vagrant/#{cachedAgentFilename}" config.vm.provision "shell", inline: "sudo apt-get update && sudo apt-get install -y unzip" end