Merge branch 'feature-ingest' of github.com:elastic/kibana into jen-huang-feature-ingest

Author: John Schulz

x-pack/legacy/plugins/epm/common/types.ts

@@ -122,6 +122,7 @@ export interface Dataset {
   ingeset_pipeline: string;
   vars: object[];
   type: string;
+  // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added.
   package: string;
 }
 

x-pack/legacy/plugins/epm/server/config.ts

@@ -34,7 +34,10 @@ export const getConfigSchema = (Joi: typeof JoiNamespace) => {
 
 const DEFAULT_CONFIG = {
   enabled: true,
-  registryUrl: 'http://package-registry.app.elstc.co',
+  // This is the staging url and should be later switched to https://epr.elastic.co for production
+  // Both are behind a CDN with caching, so upgrading the registry will not immidiately invalidate
+  // all the cached information.
+  registryUrl: 'https://epr.ea-web.elastic.dev',
 };
 
 // As of 2019, this is a Singleton because of the way JavaScript modules are specified.

x-pack/legacy/plugins/epm/server/lib/elasticsearch/ilm/install.ts

@@ -25,17 +25,11 @@ export async function policyExists(
   name: string,
   callCluster: CallESAsCurrentUser
 ): Promise<boolean> {
-  try {
-    // TODO: Figure out if there is a better way to check for an ILM policy to exist that
-    // does not throw an exception.
-    await callCluster('transport.request', {
-      method: 'GET',
-      path: '/_ilm/policy/' + name,
-    });
-    return true;
-  } catch (e) {
-    return false;
-  }
+  const response = await callCluster('transport.request', {
+    method: 'GET',
+    path: '/_ilm/policy/?filter_path=' + name,
+  });
 
-  return false;
+  // If the response contains a key, it means the policy exists
+  return Object.keys(response).length > 0;
 }

x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/__snapshots__/template.test.ts.snap

@@ -23,9 +23,10 @@ const isFields = (path: string) => {
  * The template is currently loaded with the pkgey-package-dataset
  */
 export async function installTemplates(pkg: RegistryPackage, callCluster: CallESAsCurrentUser) {
+  // If no datasets exist in this package, no templates have to be installed.
   if (!pkg.datasets) return;
 
-  const promises = pkg.datasets.map(async dataset => {
+  return pkg.datasets.map(async dataset => {
     // Fetch all assset entries for this dataset
     const assetEntries = await getAssetsData(pkg, isFields, dataset.name);
 
@@ -38,25 +39,21 @@ export async function installTemplates(pkg: RegistryPackage, callCluster: CallES
       }
     }
 
-    return installTemplate({ callCluster, fields, pkg, dataset });
+    return installTemplate({ callCluster, fields, dataset });
   });
-
-  return Promise.all(promises);
 }
 
 async function installTemplate({
   callCluster,
   fields,
-  pkg,
   dataset,
 }: {
   callCluster: CallESAsCurrentUser;
   fields: Field[];
-  pkg: RegistryPackage;
   dataset: Dataset;
 }): Promise<AssetReference> {
   const mappings = generateMappings(fields);
-  const templateName = generateTemplateName(pkg.name, dataset.name, dataset.type);
+  const templateName = generateTemplateName(dataset);
   const template = getTemplate(templateName + '-*', mappings);
   // TODO: Check return values for errors
   await callCluster('indices.putTemplate', {

x-pack/legacy/plugins/epm/server/lib/elasticsearch/template/install.ts

@@ -5,6 +5,8 @@
  */
 
 import { Field } from '../../fields/field';
+import { Dataset } from '../../../../common/types';
+import { getDatasetAssetBaseName } from '../index';
 
 export interface Template {
   order: number;
@@ -58,13 +60,8 @@ export function generateMappings(fields: Field[]): Mappings {
 /**
  * Generates the template name out of the given information
  */
-export function generateTemplateName(
-  packageName: string,
-  datasetName: string,
-  type: string
-): string {
-  // TODO: This is only a temporary name. More info like dataset type is needed to create full name
-  return type + '-' + packageName + '-' + datasetName;
+export function generateTemplateName(dataset: Dataset): string {
+  return getDatasetAssetBaseName(dataset);
 }
 
 function getBaseTemplate(mappings: Mappings): Template {
@@ -118,16 +115,6 @@ function getBaseTemplate(mappings: Mappings): Template {
             match_mapping_type: 'string',
           },
         },
-        // Example of a dynamic template
-        {
-          labels: {
-            path_match: 'labels.*',
-            mapping: {
-              type: 'keyword',
-            },
-            match_mapping_type: 'string',
-          },
-        },
       ],
       // As we define fields ahead, we don't need any automatic field detection
       // This makes sure all the fields are mapped to keyword by default to prevent mapping conflicts