[Beats Management] add more tests, update types, break out ES into it's own adapter

x-pack/package.json

@@ -29,8 +29,6 @@
     "@kbn/test": "link:../packages/kbn-test",
     "@types/boom": "^4.3.8",
     "@types/chance": "^1.0.1",
-    "@types/expect.js": "^0.3.29",
-    "@types/hapi": "15.0.1",
     "@types/jest": "^22.2.3",
     "@types/joi": "^10.4.0",
     "@types/lodash": "^3.10.0",

x-pack/plugins/beats/common/domain_types.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { ConfigurationBlockTypes } from './constants';
+
+export interface ConfigurationBlock {
+  type: ConfigurationBlockTypes;
+  block_yml: string;
+}
+
+export interface CMBeat {
+  id: string;
+  access_token: string;
+  verified_on?: string;
+  type: string;
+  version?: string;
+  host_ip: string;
+  host_name: string;
+  ephemeral_id?: string;
+  local_configuration_yml?: string;
+  tags?: string[];
+  central_configuration_yml?: string;
+  metadata?: {};
+}
+
+export interface BeatTag {
+  id: string;
+  configuration_blocks: ConfigurationBlock[];
+}

x-pack/plugins/beats/index.ts

@@ -10,18 +10,20 @@ import { initServerWithKibana } from './server/kibana.index';
 
 const DEFAULT_ENROLLMENT_TOKENS_TTL_S = 10 * 60; // 10 minutes
 
+export const config = Joi.object({
+  enabled: Joi.boolean().default(true),
+  encryptionKey: Joi.string(),
+  enrollmentTokensTtlInSeconds: Joi.number()
+    .integer()
+    .min(1)
+    .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
+}).default();
+export const configPrefix = 'xpack.beats';
+
 export function beats(kibana: any) {
   return new kibana.Plugin({
-    config: () =>
-      Joi.object({
-        enabled: Joi.boolean().default(true),
-        encryptionKey: Joi.string(),
-        enrollmentTokensTtlInSeconds: Joi.number()
-          .integer()
-          .min(1)
-          .default(DEFAULT_ENROLLMENT_TOKENS_TTL_S),
-      }).default(),
-    configPrefix: 'xpack.beats',
+    config: () => config,
+    configPrefix,
     id: PLUGIN.ID,
     require: ['kibana', 'elasticsearch', 'xpack_main'],
     init(server: any) {

x-pack/plugins/beats/readme.md

@@ -1,7 +1,15 @@
 # Documentation for Beats CM in x-pack kibana
 
-### Run tests
+### Run tests (from x-pack dir)
+
+Functional tests
 
 ```
 node scripts/jest.js plugins/beats --watch
 ```
+
+Functional API tests
+
+```
+node scripts/functional_tests  --config test/api_integration/config
+```

x-pack/plugins/beats/server/kibana.index.ts

@@ -4,11 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Server } from 'hapi';
 import { compose } from './lib/compose/kibana';
 import { initManagementServer } from './management_server';
 
-export const initServerWithKibana = (hapiServer: Server) => {
+export const initServerWithKibana = (hapiServer: any) => {
   const libs = compose(hapiServer);
   initManagementServer(libs);
 };

x-pack/plugins/beats/server/lib/adapters/beats/adapter_types.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CMBeat } from '../../../../common/domain_types';
+import { FrameworkUser } from '../framework/adapter_types';
+
+// FIXME: fix getBeatsWithIds return type
+export interface CMBeatsAdapter {
+  insert(beat: CMBeat): Promise<void>;
+  update(beat: CMBeat): Promise<void>;
+  get(id: string): any;
+  getAll(user: FrameworkUser): any;
+  getWithIds(user: FrameworkUser, beatIds: string[]): any;
+  verifyBeats(user: FrameworkUser, beatIds: string[]): any;
+  removeTagsFromBeats(
+    user: FrameworkUser,
+    removals: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]>;
+  assignTagsToBeats(
+    user: FrameworkUser,
+    assignments: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]>;
+}
+
+export interface BeatsTagAssignment {
+  beatId: string;
+  tag: string;
+  idxInRequest?: number;
+}
+
+interface BeatsReturnedTagAssignment {
+  status: number | null;
+  result?: string;
+}
+
+export interface CMAssignmentReturn {
+  assignments: BeatsReturnedTagAssignment[];
+}
+
+export interface BeatsRemovalReturn {
+  removals: BeatsReturnedTagAssignment[];
+}

x-pack/plugins/beats/server/lib/adapters/beats/elasticsearch_beats_adapter.ts

@@ -7,18 +7,18 @@
 import { flatten, get as _get, omit } from 'lodash';
 import moment from 'moment';
 import { INDEX_NAMES } from '../../../../common/constants';
-import {
-  BackendFrameworkAdapter,
-  CMBeat,
-  CMBeatsAdapter,
-  CMTagAssignment,
-  FrameworkRequest,
-} from '../../lib';
+import { CMBeat } from '../../../../common/domain_types';
+import { DatabaseAdapter } from '../database/adapter_types';
+import { BackendFrameworkAdapter } from '../framework/adapter_types';
+import { FrameworkUser } from '../framework/adapter_types';
+import { BeatsTagAssignment, CMBeatsAdapter } from './adapter_types';
 
 export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
+  private database: DatabaseAdapter;
   private framework: BackendFrameworkAdapter;
 
-  constructor(framework: BackendFrameworkAdapter) {
+  constructor(database: DatabaseAdapter, framework: BackendFrameworkAdapter) {
+    this.database = database;
     this.framework = framework;
   }
 
@@ -30,7 +30,10 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       type: '_doc',
     };
 
-    const response = await this.framework.callWithInternalUser('get', params);
+    const response = await this.database.get(
+      this.framework.internalUser,
+      params
+    );
     if (!response.found) {
       return null;
     }
@@ -44,14 +47,13 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       type: 'beat',
     };
 
-    const params = {
+    await this.database.create(this.framework.internalUser, {
       body,
       id: `beat:${beat.id}`,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-    await this.framework.callWithInternalUser('create', params);
+    });
   }
 
   public async update(beat: CMBeat) {
@@ -67,10 +69,10 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       refresh: 'wait_for',
       type: '_doc',
     };
-    return await this.framework.callWithInternalUser('index', params);
+    await this.database.index(this.framework.internalUser, params);
   }
 
-  public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
+  public async getWithIds(user: FrameworkUser, beatIds: string[]) {
     const ids = beatIds.map(beatId => `beat:${beatId}`);
 
     const params = {
@@ -81,14 +83,14 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       index: INDEX_NAMES.BEATS,
       type: '_doc',
     };
-    const response = await this.framework.callWithRequest(req, 'mget', params);
+    const response = await this.database.mget(user, params);
 
-    return get(response, 'docs', [])
+    return _get(response, 'docs', [])
       .filter((b: any) => b.found)
       .map((b: any) => b._source.beat);
   }
 
-  public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
+  public async verifyBeats(user: FrameworkUser, beatIds: string[]) {
     if (!Array.isArray(beatIds) || beatIds.length === 0) {
       return [];
     }
@@ -101,42 +103,36 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       ])
     );
 
-    const params = {
+    const response = await this.database.bulk(user, {
       _sourceInclude: ['beat.id', 'beat.verified_on'],
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
+    });
 
     return _get(response, 'items', []).map(b => ({
       ..._get(b, 'update.get._source.beat', {}),
       updateStatus: _get(b, 'update.result', 'unknown error'),
     }));
   }
 
-  public async getAll(req: FrameworkRequest) {
+  public async getAll(user: FrameworkUser) {
     const params = {
       index: INDEX_NAMES.BEATS,
       q: 'type:beat',
       type: '_doc',
     };
-    const response = await this.framework.callWithRequest(
-      req,
-      'search',
-      params
-    );
+    const response = await this.database.search(user, params);
 
-    const beats = get<any>(response, 'hits.hits', []);
+    const beats = _get<any>(response, 'hits.hits', []);
     return beats.map((beat: any) => omit(beat._source.beat, ['access_token']));
   }
 
   public async removeTagsFromBeats(
-    req: FrameworkRequest,
-    removals: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    user: FrameworkUser,
+    removals: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const body = flatten(
       removals.map(({ beatId, tag }) => {
         const script =
@@ -153,15 +149,13 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       })
     );
 
-    const params = {
+    const response = await this.database.bulk(user, {
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
-    return get<any>(response, 'items', []).map(
+    });
+    return _get<any>(response, 'items', []).map(
       (item: any, resultIdx: number) => ({
         idxInRequest: removals[resultIdx].idxInRequest,
         result: item.update.result,
@@ -171,9 +165,9 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
   }
 
   public async assignTagsToBeats(
-    req: FrameworkRequest,
-    assignments: CMTagAssignment[]
-  ): Promise<CMTagAssignment[]> {
+    user: FrameworkUser,
+    assignments: BeatsTagAssignment[]
+  ): Promise<BeatsTagAssignment[]> {
     const body = flatten(
       assignments.map(({ beatId, tag }) => {
         const script =
@@ -193,18 +187,18 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       })
     );
 
-    const params = {
+    const response = await this.database.bulk(user, {
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
-    };
-
-    const response = await this.framework.callWithRequest(req, 'bulk', params);
-    return get<any>(response, 'items', []).map((item: any, resultIdx: any) => ({
-      idxInRequest: assignments[resultIdx].idxInRequest,
-      result: item.update.result,
-      status: item.update.status,
-    }));
+    });
+    return _get<any>(response, 'items', []).map(
+      (item: any, resultIdx: any) => ({
+        idxInRequest: assignments[resultIdx].idxInRequest,
+        result: item.update.result,
+        status: item.update.status,
+      })
+    );
   }
 }