Merge pull request #6831 from BigFunger/add-data-processors-date

Add data processors date

Author: BigFunger

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui.js

@@ -1,5 +1,6 @@
 import './processor_ui_container';
 import './processor_ui_append';
 import './processor_ui_convert';
+import './processor_ui_date';
 import './processor_ui_gsub';
 import './processor_ui_set';

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui_date.js

@@ -0,0 +1,58 @@
+import uiModules from 'ui/modules';
+import template from '../views/processor_ui_date.html';
+import _ from 'lodash';
+import keysDeep from '../lib/keys_deep';
+const createMultiSelectModel = require('../lib/create_multi_select_model');
+import '../styles/_processor_ui_date.less';
+
+const app = uiModules.get('kibana');
+
+//scope.processor, scope.pipeline are attached by the process_container.
+app.directive('processorUiDate', function () {
+  return {
+    restrict: 'E',
+    template: template,
+    controller : function ($scope, debounce) {
+      const processor = $scope.processor;
+      const pipeline = $scope.pipeline;
+
+      function consumeNewInputObject() {
+        $scope.fields = keysDeep(processor.inputObject);
+        refreshFieldData();
+      }
+
+      function refreshFieldData() {
+        $scope.fieldData = _.get(processor.inputObject, processor.sourceField);
+      }
+
+      function processorUiChanged() {
+        pipeline.setDirty();
+      }
+
+      const updateFormats = debounce(() => {
+        processor.formats = _($scope.formats)
+        .filter('selected')
+        .map('title')
+        .value();
+
+        $scope.customFormatSelected = _.includes(processor.formats, 'Custom');
+        processorUiChanged();
+      }, 200);
+
+      $scope.updateFormats = updateFormats;
+      $scope.formats = createMultiSelectModel(['ISO8601', 'UNIX', 'UNIX_MS', 'TAI64N', 'Custom'], processor.formats);
+
+      $scope.$watch('processor.inputObject', consumeNewInputObject);
+
+      $scope.$watch('processor.sourceField', () => {
+        refreshFieldData();
+        processorUiChanged();
+      });
+
+      $scope.$watch('processor.customFormat', updateFormats);
+      $scope.$watch('processor.targetField', processorUiChanged);
+      $scope.$watch('processor.timezone', processorUiChanged);
+      $scope.$watch('processor.locale', processorUiChanged);
+    }
+  };
+});

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/__tests__/create_multi_select_model.js

@@ -0,0 +1,74 @@
+import expect from 'expect.js';
+import sinon from 'sinon';
+import createMultiSelectModel from '../create_multi_select_model';
+
+describe('createMultiSelectModel', function () {
+
+  it('should throw an error if the first argument is not an array', () => {
+    expect(createMultiSelectModel).withArgs('foo', []).to.throwError();
+    expect(createMultiSelectModel).withArgs(1234, []).to.throwError();
+    expect(createMultiSelectModel).withArgs(undefined, []).to.throwError();
+    expect(createMultiSelectModel).withArgs(null, []).to.throwError();
+    expect(createMultiSelectModel).withArgs([], []).to.not.throwError();
+  });
+
+  it('should throw an error if the second argument is not an array', () => {
+    expect(createMultiSelectModel).withArgs([], 'foo').to.throwError();
+    expect(createMultiSelectModel).withArgs([], 1234).to.throwError();
+    expect(createMultiSelectModel).withArgs([], undefined).to.throwError();
+    expect(createMultiSelectModel).withArgs([], null).to.throwError();
+    expect(createMultiSelectModel).withArgs([], []).to.not.throwError();
+  });
+
+  it('should output an array with an item for each passed in', () => {
+    const items = [ 'foo', 'bar', 'baz' ];
+    const expected = [
+      { title: 'foo', selected: false },
+      { title: 'bar', selected: false },
+      { title: 'baz', selected: false }
+    ];
+    const actual = createMultiSelectModel(items, []);
+
+    expect(actual).to.eql(expected);
+  });
+
+  it('should set the selected property in the output', () => {
+    const items = [ 'foo', 'bar', 'baz' ];
+    const selectedItems = [ 'bar', 'baz' ];
+    const expected = [
+      { title: 'foo', selected: false },
+      { title: 'bar', selected: true },
+      { title: 'baz', selected: true }
+    ];
+    const actual = createMultiSelectModel(items, selectedItems);
+
+    expect(actual).to.eql(expected);
+  });
+
+  it('should trim values when comparing for selected', () => {
+    const items = [ 'foo', 'bar', 'baz' ];
+    const selectedItems = [ ' bar ', ' baz ' ];
+    const expected = [
+      { title: 'foo', selected: false },
+      { title: 'bar', selected: true },
+      { title: 'baz', selected: true }
+    ];
+    const actual = createMultiSelectModel(items, selectedItems);
+
+    expect(actual).to.eql(expected);
+  });
+
+  it('should be case insensitive when comparing for selected', () => {
+    const items = [ 'foo', 'bar', 'baz' ];
+    const selectedItems = [ ' Bar ', ' BAZ ' ];
+    const expected = [
+      { title: 'foo', selected: false },
+      { title: 'bar', selected: true },
+      { title: 'baz', selected: true }
+    ];
+    const actual = createMultiSelectModel(items, selectedItems);
+
+    expect(actual).to.eql(expected);
+  });
+
+});

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/create_multi_select_model.js

@@ -0,0 +1,21 @@
+import _ from 'lodash';
+
+export default function selectableArray(items, selectedItems) {
+  if (!_.isArray(items)) throw new Error('First argument must be an array');
+  if (!_.isArray(selectedItems)) throw new Error('Second argument must be an array');
+
+  return items.map((item) => {
+    const selected = _.find(selectedItems, (selectedItem) => {
+      return cleanItem(selectedItem) === cleanItem(item);
+    });
+
+    return {
+      title: item,
+      selected: !_.isUndefined(selected)
+    };
+  });
+};
+
+function cleanItem(item) {
+  return _.trim(item).toUpperCase();
+}

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/processor_types.js

@@ -70,6 +70,37 @@ export class Convert extends Processor {
   }
 };
 
+export class Date extends Processor {
+  constructor(processorId) {
+    super(processorId, 'date', 'Date');
+    this.sourceField = '';
+    this.targetField = '@timestamp';
+    this.formats = [];
+    this.timezone = 'Etc/UTC';
+    this.locale = 'ENGLISH';
+    this.customFormat = '';
+  }
+
+  get description() {
+    const source = this.sourceField || '?';
+    const target = this.targetField || '?';
+    return `[${source}] -> [${target}]`;
+  }
+
+  get model() {
+    return {
+      processorId: this.processorId,
+      typeId: this.typeId,
+      sourceField: this.sourceField || '',
+      targetField: this.targetField || '',
+      formats: this.formats || [],
+      timezone: this.timezone || '',
+      locale: this.locale || '',
+      customFormat: this.customFormat || ''
+    };
+  }
+};
+
 export class Gsub extends Processor {
   constructor(processorId) {
     super(processorId, 'gsub', 'Gsub');

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/views/processor_ui_date.html

@@ -0,0 +1,74 @@
+<div class="form-group">
+  <label>Field:</label>
+  <select
+    class="form-control"
+    ng-options="field as field for field in fields"
+    ng-model="processor.sourceField">
+  </select>
+</div>
+<div class="form-group">
+  <label>Field Data:</label>
+  <pre>{{ fieldData }}</pre>
+</div>
+<div class="form-group">
+  <label>Target Field:</label>
+  <input type="text" class="form-control" ng-model="processor.targetField">
+</div>
+<div class="form-group">
+  <label>Formats:</label>
+  <div ng-repeat="format in formats">
+    <input
+      type="checkbox"
+      id="format_{{processor.processorId}}_{{$index}}"
+      ng-model="format.selected"
+      ng-click="updateFormats()" />
+    <label for="format_{{processor.processorId}}_{{$index}}">
+      {{format.title}}
+      <a
+        aria-label="Custom Date Format Help"
+        tooltip="Custom Date Format Help"
+        tooltip-append-to-body="true"
+        href="http://www.joda.org/joda-time/key_format.html"
+        target="_blank"
+        ng-show="format.title === 'Custom'">
+        <i aria-hidden="true" class="fa fa-question-circle"></i>
+      </a>
+    </label>
+  </div>
+  <div
+    class="custom-date-format"
+    ng-show="customFormatSelected">
+    <input
+      type="text"
+      class="form-control"
+      ng-model="processor.customFormat">
+  </div>
+</div>
+<div class="form-group">
+  <label>
+    Timezone:
+    <a
+      aria-label="Timezone Help"
+      tooltip="Timezone Help"
+      tooltip-append-to-body="true"
+      href="http://joda-time.sourceforge.net/timezones.html"
+      target="_blank">
+      <i aria-hidden="true" class="fa fa-question-circle"></i>
+    </a>
+  </label>
+  <input type="text" class="form-control" ng-model="processor.timezone"></div>
+</div>
+<div class="form-group">
+  <label>
+    Locale:
+    <a
+      aria-label="Locale Help"
+      tooltip="Locale Help"
+      tooltip-append-to-body="true"
+      href="https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html"
+      target="_blank">
+      <i aria-hidden="true" class="fa fa-question-circle"></i>
+    </a>
+  </label>
+  <input type="text" class="form-control" ng-model="processor.locale"></div>
+</div>

src/plugins/kibana/server/lib/converters/ingest_processor_api_kibana_to_es_converters.js

@@ -33,6 +33,30 @@ export function convert(processorApiDocument) {
   return processor;
 }
 
+export function date(processorApiDocument) {
+  const formats = [];
+  processorApiDocument.formats.forEach((format) => {
+    if (format.toUpperCase() === 'CUSTOM') {
+      if (processorApiDocument.custom_format) {
+        formats.push(processorApiDocument.custom_format);
+      }
+    } else {
+      formats.push(format);
+    }
+  });
+
+  return {
+    date: {
+      tag: processorApiDocument.processor_id,
+      match_field: processorApiDocument.source_field,
+      target_field: processorApiDocument.target_field,
+      match_formats: formats,
+      timezone: processorApiDocument.timezone,
+      locale: processorApiDocument.locale
+    }
+  };
+}
+
 export function gsub(processorApiDocument) {
   return {
     gsub: {

src/plugins/kibana/server/lib/schemas/resources/ingest_processor_schemas.js

@@ -17,6 +17,16 @@ export const convert = base.keys({
   type: Joi.string()
 });
 
+export const date = base.keys({
+  type_id: Joi.string().only('date').required(),
+  source_field: Joi.string().allow(''),
+  target_field: Joi.string().allow(''),
+  formats: Joi.array().items(Joi.string().allow('')),
+  timezone: Joi.string().allow(''),
+  locale: Joi.string().allow(''),
+  custom_format: Joi.string().allow('')
+});
+
 export const gsub = base.keys({
   type_id: Joi.string().only('gsub').required(),
   source_field: Joi.string().allow(''),