[Endpoint][EPM] Retrieve Index Pattern from Ingest Manager

.github/CODEOWNERS

@@ -209,9 +209,12 @@
 # Endpoint
 /x-pack/plugins/endpoint/ @elastic/endpoint-app-team
 /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/endpoint_api_integration_no_ingest/ @elastic/endpoint-app-team
 /x-pack/test/functional_endpoint/ @elastic/endpoint-app-team
 /x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team
 /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/plugin_functional/plugins/resolver_test/ @elastic/endpoint-app-team
+/x-pack/test/plugin_functional/test_suites/resolver/ @elastic/endpoint-app-team
 
 # SIEM
 /x-pack/legacy/plugins/siem/ @elastic/siem

x-pack/plugins/endpoint/common/schema/index_pattern.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const indexPatternGetParamsSchema = schema.object({ datasetPath: schema.string() });

x-pack/plugins/endpoint/common/types.ts

@@ -7,6 +7,7 @@
 import { SearchResponse } from 'elasticsearch';
 import { TypeOf } from '@kbn/config-schema';
 import { alertingIndexGetQuerySchema } from './schema/alert_index';
+import { indexPatternGetParamsSchema } from './schema/index_pattern';
 import { Datasource, NewDatasource } from '../../ingest_manager/common';
 
 /**
@@ -33,9 +34,9 @@ export type Direction = 'asc' | 'desc';
 
 export class EndpointAppConstants {
   static BASE_API_URL = '/api/endpoint';
-  static ENDPOINT_INDEX_NAME = 'endpoint-agent*';
+  static INDEX_PATTERN_ROUTE = `${EndpointAppConstants.BASE_API_URL}/index_pattern`;
   static ALERT_INDEX_NAME = 'events-endpoint-1';
-  static EVENT_INDEX_NAME = 'events-endpoint-*';
+  static EVENT_DATASET = 'events';
   static DEFAULT_TOTAL_HITS = 10000;
   /**
    * Legacy events are stored in indices with endgame-* prefix
@@ -446,6 +447,11 @@ export type AlertingIndexGetQueryInput = KbnConfigSchemaInputTypeOf<
  */
 export type AlertingIndexGetQueryResult = TypeOf<typeof alertingIndexGetQuerySchema>;
 
+/**
+ * Result of the validated params when handling an index pattern request.
+ */
+export type IndexPatternGetParamsResult = TypeOf<typeof indexPatternGetParamsSchema>;
+
 /**
  * Endpoint Policy configuration
  */

x-pack/plugins/endpoint/public/applications/endpoint/store/alerts/middleware.ts

@@ -15,10 +15,14 @@ import { EndpointAppConstants } from '../../../../../common/types';
 export const alertMiddlewareFactory: MiddlewareFactory<AlertListState> = (coreStart, depsStart) => {
   async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
     const { indexPatterns } = depsStart.data;
-    const indexName = EndpointAppConstants.ALERT_INDEX_NAME;
-    const fields = await indexPatterns.getFieldsForWildcard({ pattern: indexName });
+    const eventsPattern: { indexPattern: string } = await coreStart.http.get(
+      `${EndpointAppConstants.INDEX_PATTERN_ROUTE}/${EndpointAppConstants.EVENT_DATASET}`
+    );
+    const fields = await indexPatterns.getFieldsForWildcard({
+      pattern: eventsPattern.indexPattern,
+    });
     const indexPattern: IIndexPattern = {
-      title: indexName,
+      title: eventsPattern.indexPattern,
       fields,
     };
 

x-pack/plugins/endpoint/scripts/resolver_generator.ts

@@ -41,7 +41,7 @@ async function main() {
     metadataIndex: {
       alias: 'mi',
       describe: 'index to store host metadata in',
-      default: 'endpoint-agent-1',
+      default: 'metrics-endpoint-default-1',
       type: 'string',
     },
     auth: {

x-pack/plugins/endpoint/server/index_pattern.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { Logger, LoggerFactory, RequestHandlerContext } from 'kibana/server';
+import { ESIndexPatternService } from '../../ingest_manager/server';
+import { EndpointAppConstants } from '../common/types';
+
+export interface IndexPatternRetriever {
+  getIndexPattern(ctx: RequestHandlerContext, datasetPath: string): Promise<string>;
+  getEventIndexPattern(ctx: RequestHandlerContext): Promise<string>;
+  getMetadataIndexPattern(ctx: RequestHandlerContext): Promise<string>;
+}
+
+/**
+ * This class is used to retrieve an index pattern. It should be used in the server side code whenever
+ * an index pattern is needed to query data within ES. The index pattern is constructed by the Ingest Manager
+ * based on the contents of the Endpoint Package in the Package Registry.
+ */
+export class IngestIndexPatternRetriever implements IndexPatternRetriever {
+  private static endpointPackageName = 'endpoint';
+  private static metadataDataset = 'metadata';
+  private readonly log: Logger;
+  constructor(private readonly service: ESIndexPatternService, loggerFactory: LoggerFactory) {
+    this.log = loggerFactory.get('index-pattern-retriever');
+  }
+
+  /**
+   * Retrieves the index pattern for querying events within elasticsearch.
+   *
+   * @param ctx a RequestHandlerContext from a route handler
+   * @returns a string representing the index pattern (e.g. `events-endpoint-*`)
+   */
+  async getEventIndexPattern(ctx: RequestHandlerContext) {
+    return await this.getIndexPattern(ctx, EndpointAppConstants.EVENT_DATASET);
+  }
+
+  /**
+   * Retrieves the index pattern for querying endpoint metadata within elasticsearch.
+   *
+   * @param ctx a RequestHandlerContext from a route handler
+   * @returns a string representing the index pattern (e.g. `metrics-endpoint-*`)
+   */
+  async getMetadataIndexPattern(ctx: RequestHandlerContext) {
+    return await this.getIndexPattern(ctx, IngestIndexPatternRetriever.metadataDataset);
+  }
+
+  /**
+   * Retrieves the index pattern for a specific dataset for querying endpoint data.
+   *
+   * @param ctx a RequestHandlerContext from a route handler
+   * @param datasetPath a string of the path being used for a dataset within the Endpoint Package
+   * (e.g. `events`, `metadata`)
+   * @returns a string representing the index pattern (e.g. `metrics-endpoint-*`)
+   */
+  async getIndexPattern(ctx: RequestHandlerContext, datasetPath: string) {
+    try {
+      const pattern = await this.service.getESIndexPattern(
+        ctx.core.savedObjects.client,
+        IngestIndexPatternRetriever.endpointPackageName,
+        datasetPath
+      );
+
+      if (!pattern) {
+        const msg = `Unable to retrieve the index pattern for dataset: ${datasetPath}`;
+        this.log.warn(msg);
+        throw new Error(msg);
+      }
+      return pattern;
+    } catch (error) {
+      const errMsg = `Error occurred while retrieving pattern for: ${datasetPath} error: ${error}`;
+      this.log.warn(errMsg);
+      throw new Error(errMsg);
+    }
+  }
+}

x-pack/plugins/endpoint/server/mocks.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/**
+ * Creates a mock IndexPatternRetriever for use in tests.
+ *
+ * @param indexPattern a string index pattern to return when any of the mock's public methods are called.
+ * @returns the same string passed in via `indexPattern`
+ */
+export const createMockIndexPatternRetriever = (indexPattern: string) => {
+  const mockGetFunc = jest.fn().mockResolvedValue(indexPattern);
+  return {
+    getIndexPattern: mockGetFunc,
+    getEventIndexPattern: mockGetFunc,
+    getMetadataIndexPattern: mockGetFunc,
+  };
+};
+
+export const MetadataIndexPattern = 'metrics-endpoint-*';
+
+/**
+ * Creates a mock IndexPatternRetriever for use in tests that returns `metrics-endpoint-*`
+ */
+export const createMockMetadataIndexPatternRetriever = () => {
+  return createMockIndexPatternRetriever(MetadataIndexPattern);
+};
+
+/**
+ * Creates a mock IndexPatternService for use in tests that need to interact with the Ingest Manager's
+ * ESIndexPatternService.
+ *
+ * @param indexPattern a string index pattern to return when called by a test
+ * @returns the same value as `indexPattern` parameter
+ */
+export const createMockIndexPatternService = (indexPattern: string) => {
+  return {
+    esIndexPatternService: {
+      getESIndexPattern: jest.fn().mockResolvedValue(indexPattern),
+    },
+  };
+};

x-pack/plugins/endpoint/server/plugin.test.ts

@@ -7,6 +7,7 @@
 import { EndpointPlugin, EndpointPluginSetupDependencies } from './plugin';
 import { coreMock } from '../../../../src/core/server/mocks';
 import { PluginSetupContract } from '../../features/server';
+import { createMockIndexPatternService } from './mocks';
 
 describe('test endpoint plugin', () => {
   let plugin: EndpointPlugin;
@@ -28,7 +29,10 @@ describe('test endpoint plugin', () => {
       getFeaturesUICapabilities: jest.fn(),
       registerLegacyAPI: jest.fn(),
     };
-    mockedEndpointPluginSetupDependencies = { features: mockedPluginSetupContract };
+    mockedEndpointPluginSetupDependencies = {
+      features: mockedPluginSetupContract,
+      ingestManager: createMockIndexPatternService(''),
+    };
   });
 
   it('test properly setup plugin', async () => {

x-pack/plugins/endpoint/server/plugin.ts

@@ -6,19 +6,23 @@
 import { Plugin, CoreSetup, PluginInitializerContext, Logger } from 'kibana/server';
 import { first } from 'rxjs/operators';
 import { PluginSetupContract as FeaturesPluginSetupContract } from '../../features/server';
+import { IngestManagerSetupContract } from '../../ingest_manager/server';
 import { createConfig$, EndpointConfigType } from './config';
 import { EndpointAppContext } from './types';
 
 import { registerAlertRoutes } from './routes/alerts';
 import { registerResolverRoutes } from './routes/resolver';
+import { registerIndexPatternRoute } from './routes/index_pattern';
 import { registerEndpointRoutes } from './routes/metadata';
+import { IngestIndexPatternRetriever } from './index_pattern';
 
 export type EndpointPluginStart = void;
 export type EndpointPluginSetup = void;
 export interface EndpointPluginStartDependencies {} // eslint-disable-line @typescript-eslint/no-empty-interface
 
 export interface EndpointPluginSetupDependencies {
   features: FeaturesPluginSetupContract;
+  ingestManager: IngestManagerSetupContract;
 }
 
 export class EndpointPlugin
@@ -62,6 +66,10 @@ export class EndpointPlugin
       },
     });
     const endpointContext = {
+      indexPatternRetriever: new IngestIndexPatternRetriever(
+        plugins.ingestManager.esIndexPatternService,
+        this.initializerContext.logger
+      ),
       logFactory: this.initializerContext.logger,
       config: (): Promise<EndpointConfigType> => {
         return createConfig$(this.initializerContext)
@@ -73,6 +81,7 @@ export class EndpointPlugin
     registerEndpointRoutes(router, endpointContext);
     registerResolverRoutes(router, endpointContext);
     registerAlertRoutes(router, endpointContext);
+    registerIndexPatternRoute(router, endpointContext);
   }
 
   public start() {

x-pack/plugins/endpoint/server/routes/alerts/alerts.test.ts

@@ -12,6 +12,7 @@ import {
 import { registerAlertRoutes } from './index';
 import { EndpointConfigSchema } from '../../config';
 import { alertingIndexGetQuerySchema } from '../../../common/schema/alert_index';
+import { createMockIndexPatternRetriever } from '../../mocks';
 
 describe('test alerts route', () => {
   let routerMock: jest.Mocked<IRouter>;
@@ -24,6 +25,7 @@ describe('test alerts route', () => {
     mockClusterClient.asScoped.mockReturnValue(mockScopedClient);
     routerMock = httpServiceMock.createRouter();
     registerAlertRoutes(routerMock, {
+      indexPatternRetriever: createMockIndexPatternRetriever('events-endpoint-*'),
       logFactory: loggingServiceMock.create(),
       config: () => Promise.resolve(EndpointConfigSchema.validate({})),
     });

x-pack/plugins/endpoint/server/routes/alerts/details/handlers.ts

@@ -26,15 +26,18 @@ export const alertDetailsHandlerWrapper = function(
         id: alertId,
       })) as GetResponse<AlertEvent>;
 
+      const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx);
+
       const config = await endpointAppContext.config();
       const pagination: AlertDetailsPagination = new AlertDetailsPagination(
         config,
         ctx,
         req.params,
-        response
+        response,
+        indexPattern
       );
 
-      const currentHostInfo = await getHostData(ctx, response._source.host.id);
+      const currentHostInfo = await getHostData(ctx, response._source.host.id, indexPattern);
 
       return res.ok({
         body: {

x-pack/plugins/endpoint/server/routes/alerts/details/lib/pagination.ts

@@ -29,7 +29,8 @@ export class AlertDetailsPagination extends Pagination<
     config: EndpointConfigType,
     requestContext: RequestHandlerContext,
     state: AlertDetailsRequestParams,
-    data: GetResponse<AlertEvent>
+    data: GetResponse<AlertEvent>,
+    private readonly indexPattern: string
   ) {
     super(config, requestContext, state, data);
   }
@@ -54,7 +55,8 @@ export class AlertDetailsPagination extends Pagination<
 
     const response = await searchESForAlerts(
       this.requestContext.core.elasticsearch.dataClient,
-      reqData
+      reqData,
+      this.indexPattern
     );
     return response;
   }

x-pack/plugins/endpoint/server/routes/alerts/lib/index.ts

@@ -97,7 +97,8 @@ function buildSort(query: AlertSearchQuery): AlertSort {
  * Builds a request body for Elasticsearch, given a set of query params.
  **/
 const buildAlertSearchQuery = async (
-  query: AlertSearchQuery
+  query: AlertSearchQuery,
+  indexPattern: string
 ): Promise<AlertSearchRequestWrapper> => {
   let totalHitsMin: number = EndpointAppConstants.DEFAULT_TOTAL_HITS;
 
@@ -125,7 +126,7 @@ const buildAlertSearchQuery = async (
 
   const reqWrapper: AlertSearchRequestWrapper = {
     size: query.pageSize,
-    index: EndpointAppConstants.ALERT_INDEX_NAME,
+    index: indexPattern,
     body: reqBody,
   };
 
@@ -141,9 +142,10 @@ const buildAlertSearchQuery = async (
  **/
 export const searchESForAlerts = async (
   dataClient: IScopedClusterClient,
-  query: AlertSearchQuery
+  query: AlertSearchQuery,
+  indexPattern: string
 ): Promise<SearchResponse<AlertEvent>> => {
-  const reqWrapper = await buildAlertSearchQuery(query);
+  const reqWrapper = await buildAlertSearchQuery(query, indexPattern);
   const response = (await dataClient.callAsCurrentUser('search', reqWrapper)) as SearchResponse<
     AlertEvent
   >;

x-pack/plugins/endpoint/server/routes/alerts/list/handlers.ts

@@ -18,8 +18,13 @@ export const alertListHandlerWrapper = function(
     res
   ) => {
     try {
+      const indexPattern = await endpointAppContext.indexPatternRetriever.getEventIndexPattern(ctx);
       const reqData = await getRequestData(req, endpointAppContext);
-      const response = await searchESForAlerts(ctx.core.elasticsearch.dataClient, reqData);
+      const response = await searchESForAlerts(
+        ctx.core.elasticsearch.dataClient,
+        reqData,
+        indexPattern
+      );
       const mappedBody = await mapToAlertResultList(ctx, endpointAppContext, reqData, response);
       return res.ok({ body: mappedBody });
     } catch (err) {