[7.x] [Ingest] Encrypt secrets in fleet saved objects (#63982)

x-pack/plugins/ingest_manager/common/constants/output.ts

@@ -12,5 +12,4 @@ export const DEFAULT_OUTPUT = {
   is_default: true,
   type: OutputType.Elasticsearch,
   hosts: [''],
-  api_key: '',
 };

x-pack/plugins/ingest_manager/common/types/models/agent.ts

@@ -20,15 +20,18 @@ export interface NewAgentAction {
   sent_at?: string;
 }
 
-export type AgentAction = NewAgentAction & {
+export interface AgentAction extends NewAgentAction {
   id: string;
   agent_id: string;
   created_at: string;
-} & SavedObjectAttributes;
+}
 
-export interface AgentActionSOAttributes extends NewAgentAction, SavedObjectAttributes {
+export interface AgentActionSOAttributes extends SavedObjectAttributes {
+  type: 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE';
+  sent_at?: string;
   created_at: string;
   agent_id: string;
+  data?: string;
 }
 
 export interface AgentEvent {
@@ -64,6 +67,7 @@ interface AgentBase {
   shared_id?: string;
   access_api_key_id?: string;
   default_api_key?: string;
+  default_api_key_id?: string;
   config_id?: string;
   config_revision?: number | null;
   config_newest_revision?: number;

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -15,7 +15,10 @@ import {
 } from 'kibana/server';
 import { deepFreeze } from '../../../../src/core/utils';
 import { LicensingPluginSetup } from '../../licensing/server';
-import { EncryptedSavedObjectsPluginStart } from '../../encrypted_saved_objects/server';
+import {
+  EncryptedSavedObjectsPluginStart,
+  EncryptedSavedObjectsPluginSetup,
+} from '../../encrypted_saved_objects/server';
 import { SecurityPluginSetup } from '../../security/server';
 import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server';
 import {
@@ -28,7 +31,7 @@ import {
   AGENT_EVENT_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
 } from './constants';
-
+import { registerEncryptedSavedObjects } from './saved_objects';
 import {
   registerEPMRoutes,
   registerDatasourceRoutes,
@@ -48,6 +51,7 @@ export interface IngestManagerSetupDeps {
   licensing: LicensingPluginSetup;
   security?: SecurityPluginSetup;
   features?: FeaturesPluginSetup;
+  encryptedSavedObjects: EncryptedSavedObjectsPluginSetup;
 }
 
 export interface IngestManagerAppContext {
@@ -80,6 +84,8 @@ export class IngestManagerPlugin implements Plugin<void, IngestManagerStartupCon
       this.security = deps.security;
     }
 
+    registerEncryptedSavedObjects(deps.encryptedSavedObjects);
+
     // Register feature
     // TODO: Flesh out privileges
     if (deps.features) {

x-pack/plugins/ingest_manager/server/saved_objects.ts

@@ -13,6 +13,7 @@ import {
   AGENT_ACTION_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
 } from './constants';
+import { EncryptedSavedObjectsPluginSetup } from '../../encrypted_saved_objects/server';
 
 /*
  * Saved object mappings
@@ -35,7 +36,7 @@ export const savedObjectMappings = {
       last_checkin: { type: 'date' },
       config_revision: { type: 'integer' },
       config_newest_revision: { type: 'integer' },
-      // FIXME_INGEST https://github.com/elastic/kibana/issues/56554
+      default_api_key_id: { type: 'keyword' },
       default_api_key: { type: 'keyword' },
       updated_at: { type: 'date' },
       current_error_events: { type: 'text' },
@@ -45,8 +46,7 @@ export const savedObjectMappings = {
     properties: {
       agent_id: { type: 'keyword' },
       type: { type: 'keyword' },
-      // FIXME_INGEST https://github.com/elastic/kibana/issues/56554
-      data: { type: 'flattened' },
+      data: { type: 'binary' },
       sent_at: { type: 'date' },
       created_at: { type: 'date' },
     },
@@ -83,7 +83,6 @@ export const savedObjectMappings = {
     properties: {
       name: { type: 'keyword' },
       type: { type: 'keyword' },
-      // FIXME_INGEST https://github.com/elastic/kibana/issues/56554
       api_key: { type: 'binary' },
       api_key_id: { type: 'keyword' },
       config_id: { type: 'keyword' },
@@ -100,8 +99,6 @@ export const savedObjectMappings = {
       is_default: { type: 'boolean' },
       hosts: { type: 'keyword' },
       ca_sha256: { type: 'keyword' },
-      // FIXME_INGEST https://github.com/elastic/kibana/issues/56554
-      api_key: { type: 'keyword' },
       fleet_enroll_username: { type: 'binary' },
       fleet_enroll_password: { type: 'binary' },
       config: { type: 'flattened' },
@@ -165,3 +162,61 @@ export const savedObjectMappings = {
     },
   },
 };
+
+export function registerEncryptedSavedObjects(
+  encryptedSavedObjects: EncryptedSavedObjectsPluginSetup
+) {
+  // Encrypted saved objects
+  encryptedSavedObjects.registerType({
+    type: ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
+    attributesToEncrypt: new Set(['api_key']),
+    attributesToExcludeFromAAD: new Set([
+      'name',
+      'type',
+      'api_key_id',
+      'config_id',
+      'created_at',
+      'updated_at',
+      'expire_at',
+      'active',
+    ]),
+  });
+  encryptedSavedObjects.registerType({
+    type: OUTPUT_SAVED_OBJECT_TYPE,
+    attributesToEncrypt: new Set(['fleet_enroll_username', 'fleet_enroll_password']),
+    attributesToExcludeFromAAD: new Set([
+      'name',
+      'type',
+      'is_default',
+      'hosts',
+      'ca_sha256',
+      'config',
+    ]),
+  });
+  encryptedSavedObjects.registerType({
+    type: AGENT_SAVED_OBJECT_TYPE,
+    attributesToEncrypt: new Set(['default_api_key']),
+    attributesToExcludeFromAAD: new Set([
+      'shared_id',
+      'type',
+      'active',
+      'enrolled_at',
+      'access_api_key_id',
+      'version',
+      'user_provided_metadata',
+      'local_metadata',
+      'config_id',
+      'last_updated',
+      'last_checkin',
+      'config_revision',
+      'config_newest_revision',
+      'updated_at',
+      'current_error_events',
+    ]),
+  });
+  encryptedSavedObjects.registerType({
+    type: AGENT_ACTION_SAVED_OBJECT_TYPE,
+    attributesToEncrypt: new Set(['data']),
+    attributesToExcludeFromAAD: new Set(['agent_id', 'type', 'sent_at', 'created_at']),
+  });
+}

x-pack/plugins/ingest_manager/server/services/agents/acks.test.ts

@@ -6,6 +6,8 @@
 import Boom from 'boom';
 import { SavedObjectsBulkResponse } from 'kibana/server';
 import { savedObjectsClientMock } from '../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
+import { encryptedSavedObjectsMock } from '../../../../../plugins/encrypted_saved_objects/server/mocks';
+
 import {
   Agent,
   AgentAction,
@@ -14,10 +16,31 @@ import {
 } from '../../../common/types/models';
 import { AGENT_TYPE_PERMANENT } from '../../../common/constants';
 import { acknowledgeAgentActions } from './acks';
+import { appContextService } from '../app_context';
+import { IngestManagerAppContext } from '../../plugin';
 
 describe('test agent acks services', () => {
   it('should succeed on valid and matched actions', async () => {
     const mockSavedObjectsClient = savedObjectsClientMock.create();
+    const mockStartEncryptedSOClient = encryptedSavedObjectsMock.createStart();
+    appContextService.start(({
+      encryptedSavedObjects: mockStartEncryptedSOClient,
+    } as unknown) as IngestManagerAppContext);
+
+    mockStartEncryptedSOClient.getDecryptedAsInternalUser.mockReturnValue(
+      Promise.resolve({
+        id: 'action1',
+        references: [],
+        type: 'agent_actions',
+        attributes: {
+          type: 'CONFIG_CHANGE',
+          agent_id: 'id',
+          sent_at: '2020-03-14T19:45:02.620Z',
+          timestamp: '2019-01-04T14:32:03.36764-05:00',
+          created_at: '2020-03-14T19:45:02.620Z',
+        },
+      })
+    );
 
     mockSavedObjectsClient.bulkGet.mockReturnValue(
       Promise.resolve({

x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts

@@ -6,17 +6,17 @@
 
 import { createAgentAction } from './actions';
 import { SavedObject } from 'kibana/server';
-import { AgentAction, AgentActionSOAttributes } from '../../../common/types/models';
+import { AgentAction } from '../../../common/types/models';
 import { savedObjectsClientMock } from '../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
 
 describe('test agent actions services', () => {
   it('should create a new action', async () => {
     const mockSavedObjectsClient = savedObjectsClientMock.create();
 
-    const newAgentAction: AgentActionSOAttributes = {
+    const newAgentAction: Omit<AgentAction, 'id'> = {
       agent_id: 'agentid',
       type: 'CONFIG_CHANGE',
-      data: 'data',
+      data: { content: 'data' },
       sent_at: '2020-03-14T19:45:02.620Z',
       created_at: '2020-03-14T19:45:02.620Z',
     };
@@ -31,7 +31,7 @@ describe('test agent actions services', () => {
       .calls[0][1] as unknown) as AgentAction;
     expect(createdAction).toBeDefined();
     expect(createdAction?.type).toEqual(newAgentAction.type);
-    expect(createdAction?.data).toEqual(newAgentAction.data);
+    expect(createdAction?.data).toEqual(JSON.stringify(newAgentAction.data));
     expect(createdAction?.sent_at).toEqual(newAgentAction.sent_at);
   });
 });

x-pack/plugins/ingest_manager/server/services/agents/actions.ts

@@ -8,16 +8,21 @@ import { SavedObjectsClientContract } from 'kibana/server';
 import { Agent, AgentAction, AgentActionSOAttributes } from '../../../common/types/models';
 import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../../common/constants';
 import { savedObjectToAgentAction } from './saved_objects';
+import { appContextService } from '../app_context';
 
 export async function createAgentAction(
   soClient: SavedObjectsClientContract,
-  newAgentAction: AgentActionSOAttributes
+  newAgentAction: Omit<AgentAction, 'id'>
 ): Promise<AgentAction> {
   const so = await soClient.create<AgentActionSOAttributes>(AGENT_ACTION_SAVED_OBJECT_TYPE, {
     ...newAgentAction,
+    data: newAgentAction.data ? JSON.stringify(newAgentAction.data) : undefined,
   });
 
-  return savedObjectToAgentAction(so);
+  const agentAction = savedObjectToAgentAction(so);
+  agentAction.data = newAgentAction.data;
+
+  return agentAction;
 }
 
 export async function getAgentActionsForCheckin(
@@ -29,21 +34,47 @@ export async function getAgentActionsForCheckin(
     filter: `not ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at: * and ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.agent_id:${agentId}`,
   });
 
-  return res.saved_objects.map(savedObjectToAgentAction);
+  return Promise.all(
+    res.saved_objects.map(async so => {
+      // Get decrypted actions
+      return savedObjectToAgentAction(
+        await appContextService
+          .getEncryptedSavedObjects()
+          .getDecryptedAsInternalUser<AgentActionSOAttributes>(
+            AGENT_ACTION_SAVED_OBJECT_TYPE,
+            so.id
+          )
+      );
+    })
+  );
 }
 
 export async function getAgentActionByIds(
   soClient: SavedObjectsClientContract,
   actionIds: string[]
 ) {
-  const res = await soClient.bulkGet<AgentActionSOAttributes>(
-    actionIds.map(actionId => ({
-      id: actionId,
-      type: AGENT_ACTION_SAVED_OBJECT_TYPE,
-    }))
-  );
+  const actions = (
+    await soClient.bulkGet<AgentActionSOAttributes>(
+      actionIds.map(actionId => ({
+        id: actionId,
+        type: AGENT_ACTION_SAVED_OBJECT_TYPE,
+      }))
+    )
+  ).saved_objects.map(savedObjectToAgentAction);
 
-  return res.saved_objects.map(savedObjectToAgentAction);
+  return Promise.all(
+    actions.map(async action => {
+      // Get decrypted actions
+      return savedObjectToAgentAction(
+        await appContextService
+          .getEncryptedSavedObjects()
+          .getDecryptedAsInternalUser<AgentActionSOAttributes>(
+            AGENT_ACTION_SAVED_OBJECT_TYPE,
+            action.id
+          )
+      );
+    })
+  );
 }
 
 export interface ActionsService {

x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts

@@ -53,12 +53,12 @@ describe('Agent checkin service', () => {
             agent_id: 'agent1',
             type: 'CONFIG_CHANGE',
             created_at: new Date().toISOString(),
-            data: JSON.stringify({
+            data: {
               config: {
                 id: 'config1',
                 revision: 2,
               },
-            }),
+            },
           },
         ]
       );
@@ -80,24 +80,24 @@ describe('Agent checkin service', () => {
             agent_id: 'agent1',
             type: 'CONFIG_CHANGE',
             created_at: new Date().toISOString(),
-            data: JSON.stringify({
+            data: {
               config: {
                 id: 'config2',
                 revision: 2,
               },
-            }),
+            },
           },
           {
             id: 'action1',
             agent_id: 'agent1',
             type: 'CONFIG_CHANGE',
             created_at: new Date().toISOString(),
-            data: JSON.stringify({
+            data: {
               config: {
                 id: 'config1',
                 revision: 1,
               },
-            }),
+            },
           },
         ]
       );