EMT-339: add policy response index and documents (#65004)

* EMT-339: add policy response index and documents

* EMT-339: add routes, unit and integration tests

* EMT-339: review comments, change types, url, update tests

Author: nnamdifrankie

x-pack/plugins/endpoint/common/generate_data.test.ts

@@ -45,6 +45,17 @@ describe('data generator', () => {
     expect(metadata.host).not.toBeNull();
   });
 
+  it('creates policy response documents', () => {
+    const timestamp = new Date().getTime();
+    const hostPolicyResponse = generator.generatePolicyResponse(timestamp);
+    expect(hostPolicyResponse['@timestamp']).toEqual(timestamp);
+    expect(hostPolicyResponse.event.created).toEqual(timestamp);
+    expect(hostPolicyResponse.endpoint).not.toBeNull();
+    expect(hostPolicyResponse.agent).not.toBeNull();
+    expect(hostPolicyResponse.host).not.toBeNull();
+    expect(hostPolicyResponse.endpoint.policy.applied).not.toBeNull();
+  });
+
   it('creates alert event documents', () => {
     const timestamp = new Date().getTime();
     const alert = generator.generateAlert(timestamp);

x-pack/plugins/endpoint/common/generate_data.ts

@@ -12,9 +12,10 @@ import {
   Host,
   HostMetadata,
   HostOS,
-  PolicyData,
   HostPolicyResponse,
+  HostPolicyResponseActions,
   HostPolicyResponseActionStatus,
+  PolicyData,
 } from './types';
 import { factory as policyFactory } from './models/policy_config';
 
@@ -136,6 +137,13 @@ export class EndpointDocGenerator {
     this.commonInfo.host.ip = this.randomArray(3, () => this.randomIP());
   }
 
+  /**
+   * Creates new random policy id for the host to simulate new policy application
+   */
+  public updatePolicyId() {
+    this.commonInfo.endpoint.policy.id = this.randomChoice(POLICIES).id;
+  }
+
   private createHostData(): HostInfo {
     return {
       agent: {
@@ -498,106 +506,145 @@ export class EndpointDocGenerator {
   /**
    * Generates a Host Policy response message
    */
-  generatePolicyResponse(): HostPolicyResponse {
+  public generatePolicyResponse(ts = new Date().getTime()): HostPolicyResponse {
+    const policyVersion = this.seededUUIDv4();
     return {
-      '@timestamp': new Date().toISOString(),
+      '@timestamp': ts,
+      agent: {
+        id: this.commonInfo.agent.id,
+        version: '1.0.0-local.20200416.0',
+      },
       elastic: {
         agent: {
-          id: 'c2a9093e-e289-4c0a-aa44-8c32a414fa7a',
+          id: this.commonInfo.elastic.agent.id,
         },
       },
       ecs: {
-        version: '1.0.0',
-      },
-      event: {
-        created: '2015-01-01T12:10:30Z',
-        kind: 'policy_response',
+        version: '1.4.0',
       },
-      agent: {
-        version: '6.0.0-rc2',
-        id: '8a4f500d',
+      host: {
+        id: this.commonInfo.host.id,
       },
       endpoint: {
-        artifacts: {
-          'global-manifest': {
-            version: '1.2.3',
-            sha256: 'abcdef',
-          },
-          'endpointpe-v4-windows': {
-            version: '1.2.3',
-            sha256: 'abcdef',
-          },
-          'user-whitelist-windows': {
-            version: '1.2.3',
-            sha256: 'abcdef',
-          },
-          'global-whitelist-windows': {
-            version: '1.2.3',
-            sha256: 'abcdef',
-          },
-        },
         policy: {
           applied: {
-            version: '1.0.0',
-            id: '17d4b81d-9940-4b64-9de5-3e03ef1fb5cf',
-            status: HostPolicyResponseActionStatus.success,
+            actions: {
+              configure_elasticsearch_connection: {
+                message: 'elasticsearch comms configured successfully',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              configure_kernel: {
+                message: 'Failed to configure kernel',
+                status: HostPolicyResponseActionStatus.failure,
+              },
+              configure_logging: {
+                message: 'Successfully configured logging',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              configure_malware: {
+                message: 'Unexpected error configuring malware',
+                status: HostPolicyResponseActionStatus.failure,
+              },
+              connect_kernel: {
+                message: 'Successfully initialized minifilter',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              detect_file_open_events: {
+                message: 'Successfully stopped file open event reporting',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              detect_file_write_events: {
+                message: 'Failed to stop file write event reporting',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              detect_image_load_events: {
+                message: 'Successfuly started image load event reporting',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              detect_process_events: {
+                message: 'Successfully started process event reporting',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              download_global_artifacts: {
+                message: 'Failed to download EXE model',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              load_config: {
+                message: 'successfully parsed configuration',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              load_malware_model: {
+                message: 'Error deserializing EXE model; no valid malware model installed',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              read_elasticsearch_config: {
+                message: 'Successfully read Elasticsearch configuration',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              read_events_config: {
+                message: 'Successfully read events configuration',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              read_kernel_config: {
+                message: 'Succesfully read kernel configuration',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              read_logging_config: {
+                message: 'field (logging.debugview) not found in config',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              read_malware_config: {
+                message: 'Successfully read malware detect configuration',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              workflow: {
+                message: 'Failed to apply a portion of the configuration (kernel)',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              download_model: {
+                message: 'Failed to apply a portion of the configuration (kernel)',
+                status: HostPolicyResponseActionStatus.success,
+              },
+              ingest_events_config: {
+                message: 'Failed to apply a portion of the configuration (kernel)',
+                status: HostPolicyResponseActionStatus.success,
+              },
+            },
+            id: this.commonInfo.endpoint.policy.id,
+            policy: {
+              id: this.commonInfo.endpoint.policy.id,
+              version: policyVersion,
+            },
             response: {
               configurations: {
-                malware: {
-                  status: HostPolicyResponseActionStatus.success,
-                  concerned_actions: ['download_model', 'workflow', 'a_custom_future_action'],
-                },
                 events: {
-                  status: HostPolicyResponseActionStatus.success,
-                  concerned_actions: ['ingest_events_config', 'workflow'],
+                  concerned_actions: this.randomHostPolicyResponseActions(),
+                  status: this.randomHostPolicyResponseActionStatus(),
                 },
                 logging: {
-                  status: HostPolicyResponseActionStatus.success,
-                  concerned_actions: ['configure_elasticsearch_connection'],
-                },
-                streaming: {
-                  status: HostPolicyResponseActionStatus.success,
-                  concerned_actions: [
-                    'detect_file_open_events',
-                    'download_global_artifacts',
-                    'a_custom_future_action',
-                  ],
-                },
-              },
-              actions: {
-                download_model: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'model downloaded',
+                  concerned_actions: this.randomHostPolicyResponseActions(),
+                  status: this.randomHostPolicyResponseActionStatus(),
                 },
-                ingest_events_config: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'no action taken',
-                },
-                workflow: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'the flow worked well',
-                },
-                a_custom_future_action: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'future message',
-                },
-                configure_elasticsearch_connection: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'some message',
-                },
-                detect_file_open_events: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'some message',
+                malware: {
+                  concerned_actions: this.randomHostPolicyResponseActions(),
+                  status: this.randomHostPolicyResponseActionStatus(),
                 },
-                download_global_artifacts: {
-                  status: HostPolicyResponseActionStatus.success,
-                  message: 'some message',
+                streaming: {
+                  concerned_actions: this.randomHostPolicyResponseActions(),
+                  status: this.randomHostPolicyResponseActionStatus(),
                 },
               },
             },
+            status: this.randomHostPolicyResponseActionStatus(),
+            version: policyVersion,
           },
         },
       },
+      event: {
+        created: ts,
+        id: this.seededUUIDv4(),
+        kind: 'policy_response',
+      },
     };
   }
 
@@ -644,6 +691,34 @@ export class EndpointDocGenerator {
   private seededUUIDv4(): string {
     return uuid.v4({ random: [...this.randomNGenerator(255, 16)] });
   }
+
+  private randomHostPolicyResponseActions(): Array<keyof HostPolicyResponseActions> {
+    return this.randomArray(this.randomN(8), () =>
+      this.randomChoice([
+        'load_config',
+        'workflow',
+        'download_global_artifacts',
+        'configure_malware',
+        'read_malware_config',
+        'load_malware_model',
+        'read_kernel_config',
+        'configure_kernel',
+        'detect_process_events',
+        'detect_file_write_events',
+        'detect_file_open_events',
+        'detect_image_load_events',
+        'connect_kernel',
+      ])
+    );
+  }
+
+  private randomHostPolicyResponseActionStatus(): HostPolicyResponseActionStatus {
+    return this.randomChoice([
+      HostPolicyResponseActionStatus.failure,
+      HostPolicyResponseActionStatus.success,
+      HostPolicyResponseActionStatus.warning,
+    ]);
+  }
 }
 
 const fakeProcessNames = [

x-pack/plugins/endpoint/common/schema/policy.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { schema } from '@kbn/config-schema';
+
+export const GetPolicyResponseSchema = {
+  query: schema.object({
+    hostId: schema.string(),
+  }),
+};

x-pack/plugins/endpoint/common/types.ts

@@ -613,15 +613,15 @@ export enum HostPolicyResponseActionStatus {
 /**
  * The details of a given action
  */
-interface HostPolicyResponseActionDetails {
+export interface HostPolicyResponseActionDetails {
   status: HostPolicyResponseActionStatus;
   message: string;
 }
 
 /**
  * A known list of possible Endpoint actions
  */
-interface HostPolicyResponseActions {
+export interface HostPolicyResponseActions {
   download_model: HostPolicyResponseActionDetails;
   ingest_events_config: HostPolicyResponseActionDetails;
   workflow: HostPolicyResponseActionDetails;
@@ -642,9 +642,6 @@ interface HostPolicyResponseActions {
   read_kernel_config: HostPolicyResponseActionDetails;
   read_logging_config: HostPolicyResponseActionDetails;
   read_malware_config: HostPolicyResponseActionDetails;
-  // The list of possible Actions will change rapidly, so the below entry will allow
-  // them without us defining them here statically
-  [key: string]: HostPolicyResponseActionDetails;
 }
 
 interface HostPolicyResponseConfigurationStatus {
@@ -656,7 +653,7 @@ interface HostPolicyResponseConfigurationStatus {
  * Information about the applying of a policy to a given host
  */
 export interface HostPolicyResponse {
-  '@timestamp': string;
+  '@timestamp': number;
   elastic: {
     agent: {
       id: string;
@@ -665,29 +662,36 @@ export interface HostPolicyResponse {
   ecs: {
     version: string;
   };
+  host: {
+    id: string;
+  };
   event: {
-    created: string;
+    created: number;
     kind: string;
+    id: string;
   };
   agent: {
     version: string;
     id: string;
   };
   endpoint: {
-    artifacts: {};
     policy: {
       applied: {
         version: string;
         id: string;
         status: HostPolicyResponseActionStatus;
+        actions: Partial<HostPolicyResponseActions>;
+        policy: {
+          id: string;
+          version: string;
+        };
         response: {
           configurations: {
             malware: HostPolicyResponseConfigurationStatus;
             events: HostPolicyResponseConfigurationStatus;
             logging: HostPolicyResponseConfigurationStatus;
             streaming: HostPolicyResponseConfigurationStatus;
           };
-          actions: Partial<HostPolicyResponseActions>;
         };
       };
     };