[Beats Management] Move tokens to use JWT, add more complete test suite

packages/kbn-i18n/yarn.lock

@@ -1557,7 +1557,7 @@ rimraf@^2.6.1:
   dependencies:
     glob "^7.0.5"
 
-rxjs@^6.1.0:
+rxjs@^6.2.1:
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.2.1.tgz#246cebec189a6cbc143a3ef9f62d6f4c91813ca1"
   dependencies:

x-pack/package.json

@@ -28,11 +28,14 @@
     "@kbn/plugin-helpers": "link:../packages/kbn-plugin-helpers",
     "@kbn/test": "link:../packages/kbn-test",
     "@types/boom": "^4.3.8",
+    "@types/chance": "^1.0.1",
+    "@types/expect.js": "^0.3.29",
     "@types/hapi": "15.0.1",
     "@types/jest": "^22.2.3",
     "@types/joi": "^10.4.0",
     "@types/lodash": "^3.10.0",
     "@types/pngjs": "^3.3.0",
+    "@types/sinon": "^5.0.1",
     "abab": "^1.0.4",
     "ansicolors": "0.3.2",
     "aws-sdk": "2.2.33",
@@ -89,6 +92,8 @@
     "@kbn/ui-framework": "link:../packages/kbn-ui-framework",
     "@samverschueren/stream-to-observable": "^0.3.0",
     "@slack/client": "^4.2.2",
+    "@types/elasticsearch": "^5.0.24",
+    "@types/jsonwebtoken": "^7.2.7",
     "@types/uuid": "^3.4.3",
     "angular-paging": "2.2.1",
     "angular-resource": "1.4.9",
@@ -121,6 +126,7 @@
     "isomorphic-fetch": "2.2.1",
     "joi": "6.10.1",
     "jquery": "^3.3.1",
+    "jsonwebtoken": "^8.3.0",
     "jstimezonedetect": "1.0.5",
     "lodash": "3.10.1",
     "lodash.mean": "^4.1.0",

x-pack/plugins/beats/index.ts

@@ -15,6 +15,7 @@ export function beats(kibana: any) {
     config: () =>
       Joi.object({
         enabled: Joi.boolean().default(true),
+        encryptionKey: Joi.string(),
         enrollmentTokensTtlInSeconds: Joi.number()
           .integer()
           .min(1)

x-pack/plugins/beats/readme.md

@@ -0,0 +1,7 @@
+# Documentation for Beats CM in x-pack kibana
+
+### Run tests
+
+```
+node scripts/jest.js plugins/beats --watch
+```

x-pack/plugins/beats/server/lib/adapters/beats/elasticsearch_beats_adapter.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { flatten, get, omit } from 'lodash';
+import { flatten, get as _get, omit } from 'lodash';
 import moment from 'moment';
 import { INDEX_NAMES } from '../../../../common/constants';
 import {
@@ -35,7 +35,7 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       return null;
     }
 
-    return get(response, '_source.beat');
+    return _get(response, '_source.beat');
   }
 
   public async insert(beat: CMBeat) {
@@ -73,22 +73,6 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
   public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
     const ids = beatIds.map(beatId => `beat:${beatId}`);
 
-    const params = {
-      _source: false,
-      body: {
-        ids,
-      },
-      index: INDEX_NAMES.BEATS,
-      type: '_doc',
-    };
-    const response = await this.framework.callWithRequest(req, 'mget', params);
-    return get(response, 'docs', []);
-  }
-
-  // TODO merge with getBeatsWithIds
-  public async getVerifiedWithIds(req: FrameworkRequest, beatIds: string[]) {
-    const ids = beatIds.map(beatId => `beat:${beatId}`);
-
     const params = {
       _sourceInclude: ['beat.id', 'beat.verified_on'],
       body: {
@@ -98,7 +82,10 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
       type: '_doc',
     };
     const response = await this.framework.callWithRequest(req, 'mget', params);
-    return get(response, 'docs', []);
+
+    return get(response, 'docs', [])
+      .filter((b: any) => b.found)
+      .map((b: any) => b._source.beat);
   }
 
   public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
@@ -115,14 +102,19 @@ export class ElasticsearchBeatsAdapter implements CMBeatsAdapter {
     );
 
     const params = {
+      _sourceInclude: ['beat.id', 'beat.verified_on'],
       body,
       index: INDEX_NAMES.BEATS,
       refresh: 'wait_for',
       type: '_doc',
     };
 
     const response = await this.framework.callWithRequest(req, 'bulk', params);
-    return get(response, 'items', []);
+
+    return _get(response, 'items', []).map(b => ({
+      ..._get(b, 'update.get._source.beat', {}),
+      updateStatus: _get(b, 'update.result', 'unknown error'),
+    }));
   }
 
   public async getAll(req: FrameworkRequest) {

x-pack/plugins/beats/server/lib/adapters/beats/memory_beats_adapter.ts

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { omit } from 'lodash';
+import moment from 'moment';
+
+import {
+  CMBeat,
+  CMBeatsAdapter,
+  CMTagAssignment,
+  FrameworkRequest,
+} from '../../lib';
+
+export class MemoryBeatsAdapter implements CMBeatsAdapter {
+  private beatsDB: CMBeat[];
+
+  constructor(beatsDB: CMBeat[]) {
+    this.beatsDB = beatsDB;
+  }
+
+  public async get(id: string) {
+    return this.beatsDB.find(beat => beat.id === id);
+  }
+
+  public async insert(beat: CMBeat) {
+    this.beatsDB.push(beat);
+  }
+
+  public async update(beat: CMBeat) {
+    const beatIndex = this.beatsDB.findIndex(b => b.id === beat.id);
+
+    this.beatsDB[beatIndex] = {
+      ...this.beatsDB[beatIndex],
+      ...beat,
+    };
+  }
+
+  public async getWithIds(req: FrameworkRequest, beatIds: string[]) {
+    return this.beatsDB.filter(beat => beatIds.includes(beat.id));
+  }
+
+  public async verifyBeats(req: FrameworkRequest, beatIds: string[]) {
+    if (!Array.isArray(beatIds) || beatIds.length === 0) {
+      return [];
+    }
+
+    const verifiedOn = moment().toJSON();
+
+    this.beatsDB.forEach((beat, i) => {
+      if (beatIds.includes(beat.id)) {
+        this.beatsDB[i].verified_on = verifiedOn;
+      }
+    });
+
+    return this.beatsDB.filter(beat => beatIds.includes(beat.id));
+  }
+
+  public async getAll(req: FrameworkRequest) {
+    return this.beatsDB.map((beat: any) => omit(beat, ['access_token']));
+  }
+
+  public async removeTagsFromBeats(
+    req: FrameworkRequest,
+    removals: CMTagAssignment[]
+  ): Promise<CMTagAssignment[]> {
+    const beatIds = removals.map(r => r.beatId);
+
+    const response = this.beatsDB
+      .filter(beat => beatIds.includes(beat.id))
+      .map(beat => {
+        const tagData = removals.find(r => r.beatId === beat.id);
+        if (tagData) {
+          if (beat.tags) {
+            beat.tags = beat.tags.filter(tag => tag !== tagData.tag);
+          }
+        }
+        return beat;
+      });
+
+    return response.map<any>((item: CMBeat, resultIdx: number) => ({
+      idxInRequest: removals[resultIdx].idxInRequest,
+      result: 'updated',
+      status: 200,
+    }));
+  }
+
+  public async assignTagsToBeats(
+    req: FrameworkRequest,
+    assignments: CMTagAssignment[]
+  ): Promise<CMTagAssignment[]> {
+    const beatIds = assignments.map(r => r.beatId);
+
+    this.beatsDB.filter(beat => beatIds.includes(beat.id)).map(beat => {
+      // get tags that need to be assigned to this beat
+      const tags = assignments
+        .filter(a => a.beatId === beat.id)
+        .map((t: CMTagAssignment) => t.tag);
+
+      if (tags.length > 0) {
+        if (!beat.tags) {
+          beat.tags = [];
+        }
+        const nonExistingTags = tags.filter(
+          (t: string) => beat.tags && !beat.tags.includes(t)
+        );
+
+        if (nonExistingTags.length > 0) {
+          beat.tags = beat.tags.concat(nonExistingTags);
+        }
+      }
+      return beat;
+    });
+
+    return assignments.map<any>((item: CMTagAssignment, resultIdx: number) => ({
+      idxInRequest: assignments[resultIdx].idxInRequest,
+      result: 'updated',
+      status: 200,
+    }));
+  }
+}

x-pack/plugins/beats/server/lib/adapters/famework/kibana/kibana_framework_adapter.ts

@@ -19,18 +19,25 @@ import {
 
 export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter {
   public version: string;
-
   private server: Server;
+  private cryptoHash: string | null;
 
   constructor(hapiServer: Server) {
     this.server = hapiServer;
     this.version = hapiServer.plugins.kibana.status.plugin.version;
+    this.cryptoHash = null;
+
+    this.validateConfig();
   }
 
   public getSetting(settingPath: string) {
-    // TODO type check this properly
+    // TODO type check server properly
+    if (settingPath === 'xpack.beats.encryptionKey') {
+      // @ts-ignore
+      return this.server.config().get(settingPath) || this.cryptoHash;
+    }
     // @ts-ignore
-    return this.server.config().get(settingPath);
+    return this.server.config().get(settingPath) || this.cryptoHash;
   }
 
   public exposeStaticDir(urlPath: string, dir: string): void {
@@ -79,4 +86,17 @@ export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter {
     const fields = await callWithRequest(internalRequest, ...rest);
     return fields;
   }
+
+  private validateConfig() {
+    // @ts-ignore
+    const config = this.server.config();
+    const encryptionKey = config.get('xpack.beats.encryptionKey');
+
+    if (!encryptionKey) {
+      this.server.log(
+        'Using a default encryption key for xpack.beats.encryptionKey. It is recommended that you set xpack.beats.encryptionKey in kibana.yml with a unique token'
+      );
+      this.cryptoHash = 'xpack_beats_default_encryptionKey';
+    }
+  }
 }

x-pack/plugins/beats/server/lib/adapters/famework/kibana/testing_framework_adapter.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Client } from 'elasticsearch';
+import { Request } from 'hapi';
+import { get } from 'lodash';
+import {
+  BackendFrameworkAdapter,
+  FrameworkRequest,
+  FrameworkRouteOptions,
+  WrappableRequest,
+} from '../../../lib';
+
+interface TestSettings {
+  enrollmentTokensTtlInSeconds: number;
+  encryptionKey: string;
+}
+
+export class TestingBackendFrameworkAdapter implements BackendFrameworkAdapter {
+  public version: string;
+  private client: Client | null;
+  private settings: TestSettings;
+
+  constructor(client: Client | null, settings: TestSettings) {
+    this.client = client;
+    this.settings = settings || {
+      encryptionKey: 'something_who_cares',
+      enrollmentTokensTtlInSeconds: 10 * 60, // 10 minutes
+    };
+    this.version = 'testing';
+  }
+
+  public getSetting(settingPath: string) {
+    switch (settingPath) {
+      case 'xpack.beats.enrollmentTokensTtlInSeconds':
+        return this.settings.enrollmentTokensTtlInSeconds;
+      case 'xpack.beats.encryptionKey':
+        return this.settings.encryptionKey;
+    }
+  }
+
+  public exposeStaticDir(urlPath: string, dir: string): void {
+    // not yet testable
+  }
+
+  public registerRoute<RouteRequest extends WrappableRequest, RouteResponse>(
+    route: FrameworkRouteOptions<RouteRequest, RouteResponse>
+  ) {
+    // not yet testable
+  }
+
+  public installIndexTemplate(name: string, template: {}) {
+    if (this.client) {
+      return this.client.indices.putTemplate({
+        body: template,
+        name,
+      });
+    }
+  }
+
+  public async callWithInternalUser(esMethod: string, options: {}) {
+    const api = get<any>(this.client, esMethod);
+
+    api(options);
+
+    return await api(options);
+  }
+
+  public async callWithRequest(
+    req: FrameworkRequest<Request>,
+    esMethod: string,
+    options: {}
+  ) {
+    const api = get<any>(this.client, esMethod);
+
+    api(options);
+
+    return await api(options);
+  }
+}