[Automatic Import] add fields mapping to readme

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.test.ts

@@ -15,6 +15,7 @@ import * as PipelineModule from './pipeline';
 import { DataStream, Docs, InputType, Pipeline, Integration } from '../../common';
 import { renderPackageManifestYAML } from './build_integration';
 import yaml from 'js-yaml';
+import { createReadme } from './readme_files';
 
 const mockedDataPath = 'path';
 const mockedId = 123;
@@ -24,6 +25,7 @@ jest.mock('./data_stream');
 jest.mock('./fields');
 jest.mock('./agent');
 jest.mock('./pipeline');
+jest.mock('./readme_files');
 
 (Utils.generateUniqueId as jest.Mock).mockReturnValue(mockedId);
 
@@ -107,22 +109,11 @@ describe('buildPackage', () => {
 
     // _dev files
     expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/_dev/build`);
-    expect(Utils.createSync).toHaveBeenCalledWith(
-      `${integrationPath}/_dev/build/docs/README.md`,
-      expect.any(String)
-    );
     expect(Utils.createSync).toHaveBeenCalledWith(
       `${integrationPath}/_dev/build/build.yml`,
       expect.any(String)
     );
 
-    // Docs files
-    expect(Utils.ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`);
-    expect(Utils.createSync).toHaveBeenCalledWith(
-      `${integrationPath}/docs/README.md`,
-      expect.any(String)
-    );
-
     // Changelog file
     expect(Utils.createSync).toHaveBeenCalledWith(
       `${integrationPath}/changelog.yml`,
@@ -210,6 +201,29 @@ describe('buildPackage', () => {
       secondDataStreamDocs
     );
   });
+
+  it('Should call createReadme once', async () => {
+    jest.clearAllMocks();
+
+    const firstDataStreamFields = [
+      { name: 'name 1', description: 'description 1', type: 'type 1' },
+    ];
+
+    const secondDataStreamFields = [
+      { name: 'name 2', description: 'description 2', type: 'type 2' },
+      { name: 'name 3', description: 'description 3', type: 'type 3' },
+    ];
+
+    (FieldsModule.createFieldMapping as jest.Mock).mockReturnValueOnce(firstDataStreamFields);
+    (FieldsModule.createFieldMapping as jest.Mock).mockReturnValueOnce(secondDataStreamFields);
+
+    await buildIntegrationModule.buildPackage(testIntegration);
+
+    expect(createReadme).toHaveBeenCalledWith(integrationPath, testIntegration.name, [
+      { datastream: firstDatastreamName, fields: firstDataStreamFields },
+      { datastream: secondDatastreamName, fields: secondDataStreamFields },
+    ]);
+  });
 });
 
 describe('renderPackageManifestYAML', () => {

x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts

@@ -16,6 +16,7 @@ import { createAgentInput } from './agent';
 import { createDataStream } from './data_stream';
 import { createFieldMapping } from './fields';
 import { createPipeline } from './pipeline';
+import { createReadme } from './readme_files';
 
 const initialVersion = '1.0.0';
 
@@ -37,17 +38,27 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
   const packageDir = createDirectories(workingDir, integration, packageDirectoryName);
 
   const dataStreamsDir = joinPath(packageDir, 'data_stream');
-
+  const dataStreamFields: object[] = [];
   for (const dataStream of integration.dataStreams) {
     const dataStreamName = dataStream.name;
     const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);
 
     createDataStream(integration.name, specificDataStreamDir, dataStream);
     createAgentInput(specificDataStreamDir, dataStream.inputTypes);
     createPipeline(specificDataStreamDir, dataStream.pipeline);
-    createFieldMapping(integration.name, dataStreamName, specificDataStreamDir, dataStream.docs);
+    const fields = createFieldMapping(
+      integration.name,
+      dataStreamName,
+      specificDataStreamDir,
+      dataStream.docs
+    );
+    dataStreamFields.push({
+      datastream: dataStreamName,
+      fields,
+    });
   }
 
+  createReadme(packageDir, integration.name, dataStreamFields);
   const zipBuffer = await createZipArchive(workingDir, packageDirectoryName);
 
   removeDirSync(workingDir);
@@ -67,7 +78,6 @@ function createDirectories(
 }
 
 function createPackage(packageDir: string, integration: Integration): void {
-  createReadme(packageDir, integration);
   createChangelog(packageDir);
   createBuildFile(packageDir);
   createPackageManifest(packageDir, integration);
@@ -102,20 +112,6 @@ function createChangelog(packageDir: string): void {
   createSync(joinPath(packageDir, 'changelog.yml'), changelogTemplate);
 }
 
-function createReadme(packageDir: string, integration: Integration) {
-  const readmeDirPath = joinPath(packageDir, '_dev/build/docs/');
-  const mainReadmeDirPath = joinPath(packageDir, 'docs/');
-  ensureDirSync(mainReadmeDirPath);
-  ensureDirSync(readmeDirPath);
-  const readmeTemplate = nunjucks.render('package_readme.md.njk', {
-    package_name: integration.name,
-    data_streams: integration.dataStreams,
-  });
-
-  createSync(joinPath(readmeDirPath, 'README.md'), readmeTemplate);
-  createSync(joinPath(mainReadmeDirPath, 'README.md'), readmeTemplate);
-}
-
 async function createZipArchive(workingDir: string, packageDirectoryName: string): Promise<Buffer> {
   const tmpPackageDir = joinPath(workingDir, packageDirectoryName);
   const zip = new AdmZip();

x-pack/plugins/integration_assistant/server/integration_builder/fields.test.ts

@@ -71,4 +71,50 @@ describe('createFieldMapping', () => {
       expectedFields
     );
   });
+
+  it('Should return all fields flattened', async () => {
+    const docs: Docs = [
+      {
+        key: 'foo',
+        anotherKey: 'bar',
+      },
+    ];
+
+    const basedFields = `- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.
+`;
+    (nunjucks.render as jest.Mock).mockReturnValue(basedFields);
+
+    const fieldsResult = createFieldMapping(packageName, dataStreamName, dataStreamPath, docs);
+
+    expect(fieldsResult).toEqual([
+      { name: '@timestamp', type: 'date', description: 'Event timestamp.' },
+      {
+        description: undefined,
+        name: 'anotherKey',
+        type: 'keyword',
+      },
+      {
+        description: undefined,
+        name: 'data_stream.dataset',
+        type: 'constant_keyword',
+      },
+      {
+        description: 'Data stream type.',
+        name: 'data_stream.type',
+        type: 'constant_keyword',
+      },
+      {
+        description: undefined,
+        name: 'key',
+        type: 'keyword',
+      },
+    ]);
+  });
 });

x-pack/plugins/integration_assistant/server/integration_builder/fields.ts

@@ -6,35 +6,48 @@
  */
 
 import nunjucks from 'nunjucks';
-
+import { load } from 'js-yaml';
+import { flattenObjectsList } from '../util/samples';
 import { createSync, generateFields, mergeSamples } from '../util';
+import { Docs } from '../../common';
 
 export function createFieldMapping(
   packageName: string,
   dataStreamName: string,
   specificDataStreamDir: string,
   docs: object[]
-): void {
-  createBaseFields(specificDataStreamDir, packageName, dataStreamName);
-  createCustomFields(specificDataStreamDir, docs);
+): object[] {
+  const baseFields = createBaseFields(specificDataStreamDir, packageName, dataStreamName);
+  const customFields = createCustomFields(specificDataStreamDir, docs);
+
+  return mergeFields(baseFields, customFields);
 }
 
 function createBaseFields(
   specificDataStreamDir: string,
   packageName: string,
   dataStreamName: string
-): void {
+): Docs[] {
   const datasetName = `${packageName}.${dataStreamName}`;
   const baseFields = nunjucks.render('base_fields.yml.njk', {
     module: packageName,
     dataset: datasetName,
   });
-
   createSync(`${specificDataStreamDir}/base-fields.yml`, baseFields);
+
+  return load(baseFields) as Docs[];
 }
 
-function createCustomFields(specificDataStreamDir: string, pipelineResults: object[]): void {
+function createCustomFields(specificDataStreamDir: string, pipelineResults: object[]): Docs[] {
   const mergedResults = mergeSamples(pipelineResults);
   const fieldKeys = generateFields(mergedResults);
   createSync(`${specificDataStreamDir}/fields/fields.yml`, fieldKeys);
+
+  return load(fieldKeys) as Docs[];
+}
+
+function mergeFields(baseFields: object[], customFields: object[]): object[] {
+  const fields = [...baseFields, ...customFields];
+
+  return flattenObjectsList(fields);
 }

x-pack/plugins/integration_assistant/server/integration_builder/readme_files.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { testIntegration } from '../../__jest__/fixtures/build_integration';
+import { ensureDirSync, createSync } from '../util';
+import { configure } from 'nunjucks';
+import { join as joinPath } from 'path';
+import { createReadme } from './readme_files';
+
+jest.mock('../util', () => ({
+  ...jest.requireActual('../util'),
+  createSync: jest.fn(),
+  ensureDirSync: jest.fn(),
+}));
+
+describe('createReadme', () => {
+  const integrationPath = 'path';
+
+  const templateDir = joinPath(__dirname, '../templates');
+  configure([templateDir], {
+    autoescape: false,
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+  });
+
+  it('Should create expected files', async () => {
+    const fields = [
+      {
+        datastream: 'data_stream_1',
+        fields: [
+          {
+            name: 'data_stream.type',
+            type: 'constant_keyword',
+            description: 'Data stream type.',
+          },
+          {
+            name: 'data_stream.dataset',
+            type: 'constant_keyword',
+            description: 'Data stream dataset name.',
+          },
+          {
+            name: 'event.dataset',
+            type: 'constant_keyword',
+            description: 'Event dataset',
+            value: 'package.datastream',
+          },
+          { name: '@timestamp', type: 'date', description: 'Event timestamp.' },
+        ],
+      },
+      {
+        datastream: 'data_stream_2',
+        fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }],
+      },
+    ];
+
+    createReadme(integrationPath, testIntegration.name, fields);
+
+    expect(createSync).toHaveBeenCalledWith(
+      `${integrationPath}/_dev/build/docs/README.md`,
+      expect.any(String)
+    );
+
+    // Docs files
+    expect(ensureDirSync).toHaveBeenCalledWith(`${integrationPath}/docs/`);
+    expect(createSync).toHaveBeenCalledWith(
+      `${integrationPath}/docs/README.md`,
+      expect.any(String)
+    );
+  });
+
+  it('Should render a table per datastream with fields mapping in package readme', async () => {
+    const fields = [
+      {
+        datastream: 'data_stream_1',
+        fields: [
+          {
+            name: 'data_stream.type',
+            type: 'constant_keyword',
+            description: 'Data stream type.',
+          },
+          {
+            name: 'data_stream.dataset',
+            type: 'constant_keyword',
+          },
+          {
+            name: 'event.dataset',
+            type: 'constant_keyword',
+            description: 'Event dataset',
+            value: 'package.datastream',
+          },
+          { name: '@timestamp', type: 'date', description: 'Event timestamp.' },
+        ],
+      },
+      {
+        datastream: 'data_stream_2',
+        fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }],
+      },
+    ];
+
+    createReadme(integrationPath, testIntegration.name, fields);
+
+    const firstDatastreamFieldsDisplayed = `
+| Field | Description | Type |
+|---|---|---|
+| data_stream.type | Data stream type. | constant_keyword |
+| data_stream.dataset | Insert a description | constant_keyword |
+| event.dataset | Event dataset | constant_keyword |
+| @timestamp | Event timestamp. | date |
+`;
+
+    const secondDatastreamFieldsDisplayed = `
+| Field | Description | Type |
+|---|---|---|
+| @timestamp | Event timestamp. | date |
+`;
+
+    expect(createSync).toHaveBeenCalledWith(
+      `${integrationPath}/docs/README.md`,
+      expect.stringContaining(firstDatastreamFieldsDisplayed)
+    );
+
+    expect(createSync).toHaveBeenCalledWith(
+      `${integrationPath}/docs/README.md`,
+      expect.stringContaining(secondDatastreamFieldsDisplayed)
+    );
+  });
+
+  it('Should not render a fields mapping table in build readme', async () => {
+    const fields = [
+      {
+        datastream: 'data_stream_1',
+        fields: [{ name: '@timestamp', type: 'date', description: 'Event timestamp.' }],
+      },
+    ];
+
+    createReadme(integrationPath, testIntegration.name, fields);
+
+    expect(createSync).toHaveBeenCalledWith(
+      `${integrationPath}/_dev/build/docs/README.md`,
+      expect.stringContaining('{{fields "data_stream_1"}}')
+    );
+  });
+});