From 9d8659d376a4e87f43b83e4ecf101b0d025d4a84 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 23 Apr 2020 08:55:33 -0400 Subject: [PATCH] [Ingest] Encrypt secrets in fleet saved objects (#63982) --- .../ingest_manager/common/constants/output.ts | 1 - .../common/types/models/agent.ts | 10 ++- .../plugins/ingest_manager/server/plugin.ts | 10 ++- .../ingest_manager/server/saved_objects.ts | 67 +++++++++++++++++-- .../server/services/agents/acks.test.ts | 23 +++++++ .../server/services/agents/actions.test.ts | 8 +-- .../server/services/agents/actions.ts | 51 +++++++++++--- .../server/services/agents/checkin.test.ts | 12 ++-- .../server/services/agents/checkin.ts | 17 +++-- .../server/services/agents/saved_objects.ts | 1 + .../server/services/agents/unenroll.ts | 7 +- .../services/api_keys/enrollment_api_key.ts | 20 ++++-- .../server/services/app_context.ts | 3 + .../apis/fleet/unenroll_agent.ts | 1 + 14 files changed, 181 insertions(+), 50 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/output.ts b/x-pack/plugins/ingest_manager/common/constants/output.ts index 6060a2b63fc8e..4c22d0e3fe7a3 100644 --- a/x-pack/plugins/ingest_manager/common/constants/output.ts +++ b/x-pack/plugins/ingest_manager/common/constants/output.ts @@ -12,5 +12,4 @@ export const DEFAULT_OUTPUT = { is_default: true, type: OutputType.Elasticsearch, hosts: [''], - api_key: '', }; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent.ts b/x-pack/plugins/ingest_manager/common/types/models/agent.ts index 14b2b2e47d17f..fcd3955f3a32f 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent.ts @@ -20,15 +20,18 @@ export interface NewAgentAction { sent_at?: string; } -export type AgentAction = NewAgentAction & { +export interface AgentAction extends NewAgentAction { id: string; agent_id: string; created_at: string; -} & SavedObjectAttributes; +} -export interface AgentActionSOAttributes extends NewAgentAction, SavedObjectAttributes { +export interface AgentActionSOAttributes extends SavedObjectAttributes { + type: 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE'; + sent_at?: string; created_at: string; agent_id: string; + data?: string; } export interface AgentEvent { @@ -64,6 +67,7 @@ interface AgentBase { shared_id?: string; access_api_key_id?: string; default_api_key?: string; + default_api_key_id?: string; config_id?: string; config_revision?: number | null; config_newest_revision?: number; diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index 80e35eadb4603..a01cfd6e90589 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -15,7 +15,10 @@ import { } from 'kibana/server'; import { deepFreeze } from '../../../../src/core/utils'; import { LicensingPluginSetup } from '../../licensing/server'; -import { EncryptedSavedObjectsPluginStart } from '../../encrypted_saved_objects/server'; +import { + EncryptedSavedObjectsPluginStart, + EncryptedSavedObjectsPluginSetup, +} from '../../encrypted_saved_objects/server'; import { SecurityPluginSetup } from '../../security/server'; import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; import { @@ -28,7 +31,7 @@ import { AGENT_EVENT_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, } from './constants'; - +import { registerEncryptedSavedObjects } from './saved_objects'; import { registerEPMRoutes, registerDatasourceRoutes, @@ -47,6 +50,7 @@ export interface IngestManagerSetupDeps { licensing: LicensingPluginSetup; security?: SecurityPluginSetup; features?: FeaturesPluginSetup; + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; } export interface IngestManagerAppContext { @@ -79,6 +83,8 @@ export class IngestManagerPlugin implements Plugin { it('should succeed on valid and matched actions', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); + const mockStartEncryptedSOClient = encryptedSavedObjectsMock.createStart(); + appContextService.start(({ + encryptedSavedObjects: mockStartEncryptedSOClient, + } as unknown) as IngestManagerAppContext); + + mockStartEncryptedSOClient.getDecryptedAsInternalUser.mockReturnValue( + Promise.resolve({ + id: 'action1', + references: [], + type: 'agent_actions', + attributes: { + type: 'CONFIG_CHANGE', + agent_id: 'id', + sent_at: '2020-03-14T19:45:02.620Z', + timestamp: '2019-01-04T14:32:03.36764-05:00', + created_at: '2020-03-14T19:45:02.620Z', + }, + }) + ); mockSavedObjectsClient.bulkGet.mockReturnValue( Promise.resolve({ diff --git a/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts index f2e671c6dbaa8..29143502247aa 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts @@ -6,17 +6,17 @@ import { createAgentAction } from './actions'; import { SavedObject } from 'kibana/server'; -import { AgentAction, AgentActionSOAttributes } from '../../../common/types/models'; +import { AgentAction } from '../../../common/types/models'; import { savedObjectsClientMock } from '../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; describe('test agent actions services', () => { it('should create a new action', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); - const newAgentAction: AgentActionSOAttributes = { + const newAgentAction: Omit = { agent_id: 'agentid', type: 'CONFIG_CHANGE', - data: 'data', + data: { content: 'data' }, sent_at: '2020-03-14T19:45:02.620Z', created_at: '2020-03-14T19:45:02.620Z', }; @@ -31,7 +31,7 @@ describe('test agent actions services', () => { .calls[0][1] as unknown) as AgentAction; expect(createdAction).toBeDefined(); expect(createdAction?.type).toEqual(newAgentAction.type); - expect(createdAction?.data).toEqual(newAgentAction.data); + expect(createdAction?.data).toEqual(JSON.stringify(newAgentAction.data)); expect(createdAction?.sent_at).toEqual(newAgentAction.sent_at); }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/agents/actions.ts b/x-pack/plugins/ingest_manager/server/services/agents/actions.ts index a8ef0820f8d9f..1bb177e54282d 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/actions.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/actions.ts @@ -8,16 +8,21 @@ import { SavedObjectsClientContract } from 'kibana/server'; import { Agent, AgentAction, AgentActionSOAttributes } from '../../../common/types/models'; import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../../common/constants'; import { savedObjectToAgentAction } from './saved_objects'; +import { appContextService } from '../app_context'; export async function createAgentAction( soClient: SavedObjectsClientContract, - newAgentAction: AgentActionSOAttributes + newAgentAction: Omit ): Promise { const so = await soClient.create(AGENT_ACTION_SAVED_OBJECT_TYPE, { ...newAgentAction, + data: newAgentAction.data ? JSON.stringify(newAgentAction.data) : undefined, }); - return savedObjectToAgentAction(so); + const agentAction = savedObjectToAgentAction(so); + agentAction.data = newAgentAction.data; + + return agentAction; } export async function getAgentActionsForCheckin( @@ -29,21 +34,47 @@ export async function getAgentActionsForCheckin( filter: `not ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at: * and ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.agent_id:${agentId}`, }); - return res.saved_objects.map(savedObjectToAgentAction); + return Promise.all( + res.saved_objects.map(async so => { + // Get decrypted actions + return savedObjectToAgentAction( + await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + AGENT_ACTION_SAVED_OBJECT_TYPE, + so.id + ) + ); + }) + ); } export async function getAgentActionByIds( soClient: SavedObjectsClientContract, actionIds: string[] ) { - const res = await soClient.bulkGet( - actionIds.map(actionId => ({ - id: actionId, - type: AGENT_ACTION_SAVED_OBJECT_TYPE, - })) - ); + const actions = ( + await soClient.bulkGet( + actionIds.map(actionId => ({ + id: actionId, + type: AGENT_ACTION_SAVED_OBJECT_TYPE, + })) + ) + ).saved_objects.map(savedObjectToAgentAction); - return res.saved_objects.map(savedObjectToAgentAction); + return Promise.all( + actions.map(async action => { + // Get decrypted actions + return savedObjectToAgentAction( + await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + AGENT_ACTION_SAVED_OBJECT_TYPE, + action.id + ) + ); + }) + ); } export interface ActionsService { diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts index ec10ca6e77e05..72a86d7c8158e 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts @@ -53,12 +53,12 @@ describe('Agent checkin service', () => { agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config1', revision: 2, }, - }), + }, }, ] ); @@ -80,24 +80,24 @@ describe('Agent checkin service', () => { agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config2', revision: 2, }, - }), + }, }, { id: 'action1', agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config1', revision: 1, }, - }), + }, }, ] ); diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts index 2873aad7f691a..c96a81ed9b758 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts @@ -17,6 +17,7 @@ import { agentConfigService } from '../agent_config'; import * as APIKeysService from '../api_keys'; import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../constants'; import { getAgentActionsForCheckin, createAgentAction } from './actions'; +import { appContextService } from '../app_context'; export async function agentCheckin( soClient: SavedObjectsClientContract, @@ -27,7 +28,6 @@ export async function agentCheckin( const updateData: { last_checkin: string; default_api_key?: string; - actions?: AgentAction[]; local_metadata?: string; current_error_events?: string; } = { @@ -38,11 +38,17 @@ export async function agentCheckin( // Generate new agent config if config is updated if (agent.config_id && shouldCreateConfigAction(agent, actions)) { + const { + attributes: { default_api_key: defaultApiKey }, + } = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, agent.id); + const config = await agentConfigService.getFullConfig(soClient, agent.config_id); if (config) { // Assign output API keys // We currently only support default ouput - if (!agent.default_api_key) { + if (!defaultApiKey) { updateData.default_api_key = await APIKeysService.generateOutputApiKey( soClient, 'default', @@ -50,7 +56,7 @@ export async function agentCheckin( ); } // Mutate the config to set the api token for this agent - config.outputs.default.api_key = agent.default_api_key || updateData.default_api_key; + config.outputs.default.api_key = defaultApiKey || updateData.default_api_key; const configChangeAction = await createAgentAction(soClient, { agent_id: agent.id, @@ -62,9 +68,6 @@ export async function agentCheckin( actions.push(configChangeAction); } } - if (localMetadata) { - updateData.local_metadata = JSON.stringify(localMetadata); - } const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, events); @@ -172,7 +175,7 @@ export function shouldCreateConfigAction(agent: Agent, actions: AgentAction[]): return false; } - const data = JSON.parse(action.data); + const { data } = action; return ( data.config.id === agent.config_id && data.config.revision === agent.config_newest_revision diff --git a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts index aa88520740687..b182662e0fb4e 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts @@ -38,5 +38,6 @@ export function savedObjectToAgentAction(so: SavedObject(AGENT_SAVED_OBJECT_TYPE, agentId, { diff --git a/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts index a6a2db8be4e9d..c9ead09b0908d 100644 --- a/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts @@ -10,6 +10,7 @@ import { EnrollmentAPIKey, EnrollmentAPIKeySOAttributes } from '../../types'; import { ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE } from '../../constants'; import { createAPIKey, invalidateAPIKey } from './security'; import { agentConfigService } from '../agent_config'; +import { appContextService } from '../app_context'; export async function listEnrollmentApiKeys( soClient: SavedObjectsClientContract, @@ -45,9 +46,13 @@ export async function listEnrollmentApiKeys( } export async function getEnrollmentAPIKey(soClient: SavedObjectsClientContract, id: string) { - return savedObjectToEnrollmentApiKey( - await soClient.get(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, id) - ); + const so = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + id + ); + return savedObjectToEnrollmentApiKey(so); } /** @@ -120,16 +125,19 @@ export async function generateEnrollmentAPIKey( const apiKey = Buffer.from(`${key.id}:${key.api_key}`).toString('base64'); - return savedObjectToEnrollmentApiKey( - await soClient.create(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, { + const so = await soClient.create( + ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + { active: true, api_key_id: key.id, api_key: apiKey, name, config_id: configId, created_at: new Date().toISOString(), - }) + } ); + + return getEnrollmentAPIKey(soClient, so.id); } function savedObjectToEnrollmentApiKey({ diff --git a/x-pack/plugins/ingest_manager/server/services/app_context.ts b/x-pack/plugins/ingest_manager/server/services/app_context.ts index a0a7c8dd7c05a..e917d2edd1309 100644 --- a/x-pack/plugins/ingest_manager/server/services/app_context.ts +++ b/x-pack/plugins/ingest_manager/server/services/app_context.ts @@ -34,6 +34,9 @@ class AppContextService { public stop() {} public getEncryptedSavedObjects() { + if (!this.encryptedSavedObjects) { + throw new Error('Encrypted saved object start service not set.'); + } return this.encryptedSavedObjects; } diff --git a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts index 2acfca63995f1..d33b92acf95a5 100644 --- a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts +++ b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts @@ -44,6 +44,7 @@ export default function(providerContext: FtrProviderContext) { }); // @ts-ignore agentDoc.agents.access_api_key_id = accessAPIKeyId; + agentDoc.agents.default_api_key_id = outputAPIKeyBody.id; agentDoc.agents.default_api_key = Buffer.from( `${outputAPIKeyBody.id}:${outputAPIKeyBody.api_key}` ).toString('base64');