From 729c48a327df48682ca69811f1ec9ddf91c00b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Campinas?= Date: Tue, 12 Jan 2016 10:08:35 +0000 Subject: [PATCH 1/8] added source filtering --- src/plugins/elasticsearch/index.js | 3 +- .../indices/__tests__/retrieved_field.js | 60 +++++++++ .../settings/sections/indices/_edit.html | 3 +- .../public/settings/sections/indices/_edit.js | 1 + .../settings/sections/indices/_field_types.js | 3 + .../sections/indices/_indexed_fields.js | 6 + .../sections/indices/_source_filtering.html | 114 ++++++++++++++++++ .../sections/indices/_source_filtering.js | 44 +++++++ .../sections/indices/retrieved_field.js | 50 ++++++++ src/testUtils/stub_index_pattern.js | 1 + .../public/doc_table/__tests__/doc_table.js | 4 + src/ui/public/doc_table/doc_table.js | 5 + .../public/index_patterns/_index_pattern.js | 15 ++- 13 files changed, 306 insertions(+), 3 deletions(-) create mode 100644 src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js create mode 100644 src/plugins/kibana/public/settings/sections/indices/_source_filtering.html create mode 100644 src/plugins/kibana/public/settings/sections/indices/_source_filtering.js create mode 100644 src/plugins/kibana/public/settings/sections/indices/retrieved_field.js diff --git a/src/plugins/elasticsearch/index.js b/src/plugins/elasticsearch/index.js index 63950a737fb54..c2346bd01ef44 100644 --- a/src/plugins/elasticsearch/index.js +++ b/src/plugins/elasticsearch/index.js @@ -24,7 +24,8 @@ module.exports = function (kibana) { key: Joi.string() }).default(), apiVersion: Joi.string().default('2.0'), - engineVersion: Joi.string().valid('^2.1.0').default('^2.1.0') + engineVersion: Joi.string().valid('^2.1.0').default('^2.1.0'), + plugins: Joi.array() }).default(); }, diff --git a/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js b/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js new file mode 100644 index 0000000000000..0543e77901163 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js @@ -0,0 +1,60 @@ +var expect = require('expect.js'); + +define(function (require) { + var isRetrieved = require('src/plugins/kibana/public/settings/sections/indices/retrieved_field'); + + describe('Settings', function () { + describe('Indices', function () { + describe('isRetrieved(sourceFiltering, name)', function () { + it('should be a function', function () { + expect(isRetrieved).to.be.a(Function); + }); + + it('should retrieve john', function () { + var sourceFiltering = { + include: 'john' + }; + + expect(isRetrieved(sourceFiltering, 'john')).to.be(true); + }); + + it('should not retrieve connor', function () { + var sourceFiltering = { + exclude: 'connor' + }; + + expect(isRetrieved(sourceFiltering, 'connor')).to.be(false); + }); + + it('should retrieve connor', function () { + var sourceFiltering = { + exclude: '*.connor' + }; + + expect(isRetrieved(sourceFiltering, 'connor')).to.be(true); + expect(isRetrieved(sourceFiltering, 'john.connor')).to.be(false); + }); + + it('should not retrieve neither john nor connor', function () { + var sourceFiltering = { + exclude: [ 'john', 'connor' ] + }; + + expect(isRetrieved(sourceFiltering, 'connor')).to.be(false); + expect(isRetrieved(sourceFiltering, 'john')).to.be(false); + expect(isRetrieved(sourceFiltering, 'toto')).to.be(true); + }); + + it('should not retrieve john.*.connor', function () { + var sourceFiltering = { + exclude: 'john.*.connor' + }; + + expect(isRetrieved(sourceFiltering, 'john.j.connor')).to.be(false); + expect(isRetrieved(sourceFiltering, 'john.t.connor')).to.be(false); + expect(isRetrieved(sourceFiltering, 'john.j.watterson')).to.be(true); + }); + }); + }); + }); +}); diff --git a/src/plugins/kibana/public/settings/sections/indices/_edit.html b/src/plugins/kibana/public/settings/sections/indices/_edit.html index ee0e6f65a307c..0c865be5a7954 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_edit.html +++ b/src/plugins/kibana/public/settings/sections/indices/_edit.html @@ -41,13 +41,14 @@
  • {{ fieldType.title }} - ({{ fieldType.count }}) + ({{ fieldType.count }})
    • + diff --git a/src/plugins/kibana/public/settings/sections/indices/_edit.js b/src/plugins/kibana/public/settings/sections/indices/_edit.js index abd5ee43ea8a9..8b7797fd1d470 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_edit.js +++ b/src/plugins/kibana/public/settings/sections/indices/_edit.js @@ -3,6 +3,7 @@ define(function (require) { require('plugins/kibana/settings/sections/indices/_indexed_fields'); require('plugins/kibana/settings/sections/indices/_scripted_fields'); require('plugins/kibana/settings/sections/indices/_index_header'); + require('plugins/kibana/settings/sections/indices/_source_filtering'); require('ui/routes') .when('/settings/indices/:indexPatternId', { diff --git a/src/plugins/kibana/public/settings/sections/indices/_field_types.js b/src/plugins/kibana/public/settings/sections/indices/_field_types.js index 6e4e1f284315c..6e87c1a691250 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_field_types.js +++ b/src/plugins/kibana/public/settings/sections/indices/_field_types.js @@ -20,6 +20,9 @@ define(function (require) { title: 'scripted fields', index: 'scriptedFields', count: fieldCount.scripted + }, { + title: 'Retrieved Fields', + index: 'sourceFiltering' }]; }; }; diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index cebb4f10e6009..4aadaf598c869 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -1,5 +1,6 @@ define(function (require) { var _ = require('lodash'); + var isRetrieved = require('./retrieved_field'); require('ui/paginated_table'); require('ui/modules').get('apps/settings') @@ -24,6 +25,7 @@ define(function (require) { { title: 'format' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, + { title: 'retrieved', info: 'Fields that are not retrieved as part of the _source object per hit' }, { title: 'controls', sortable: false } ]; @@ -33,6 +35,7 @@ define(function (require) { // clear and destroy row scopes _.invoke(rowScopes.splice(0), '$destroy'); + var sourceFiltering = $scope.indexPattern.getSourceFiltering(); var fields = filter($scope.indexPattern.getNonScriptedFields(), $scope.fieldFilter); _.find($scope.fieldTypes, {index: 'indexedFields'}).count = fields.length; // Update the tab count @@ -60,6 +63,9 @@ define(function (require) { markup: field.indexed ? yesTemplate : noTemplate, value: field.indexed }, + { + markup: isRetrieved(sourceFiltering, field.displayName) ? yesTemplate : noTemplate + }, { markup: controlsHtml, scope: childScope diff --git a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html new file mode 100644 index 0000000000000..8af8c30333912 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html @@ -0,0 +1,114 @@ +

    Retrieved fields + + +

    Retrieved Fields Help

    +     +

    + +
    +

    + Retrieved Fields Help +

    + +

    + All fields are by default retrieved and are available inside the "_source" object of each hit. Thanks to the + + source filtering API + + + of Elasticsearch, you can choose which fields are actually retrieved. The value needs to be a JSON object. +

    + +
    + Examples + +

    Exclusion of a single field

    + +
    { + "exclude": "user" +}
    + +

    Exclusion with a path pattern

    +
    { + "exclude": [ + "obj1.*", + "*.value" + ] +}
    + +

    Complete control with exclusions and inclusion

    +
    { + "exclude": "obj1.*", + "include": "obj2.*.val" +}
    +
    + +

    +By default, all fields are retrieved to populate results table. +Sometimes however some fields can be very large and seriously affect performance. This can be often the case when you have materialized one to many relationships, e.g., an entity having many nested entities. Those fields are still useful for searching or analytics but not in a result table.

    +Use this setting to decide what to include or exclude in the data retrieved from the Elasticsearch index.
    +If empty, all fields are retrieved. +

    + +
    +
    {{ sourceFiltering | json }}
    + +
    + +
    +
    diff --git a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js new file mode 100644 index 0000000000000..5026857d6cd26 --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js @@ -0,0 +1,44 @@ +define(function (require) { + require('ui/modules').get('apps/settings') + .directive('sourceFiltering', function (Notifier, $window) { + var notify = new Notifier(); + return { + restrict: 'E', + template: require('plugins/kibana/settings/sections/indices/_source_filtering.html'), + link: function ($scope) { + $scope.showHelp = false; + $scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); + $scope.save = function () { + try { + var sourceFiltering; + + if ($scope.sourceFiltering) { + sourceFiltering = JSON.parse($scope.sourceFiltering); + if (sourceFiltering.constructor !== Object) { + throw 'You must enter a JSON object with exclude/include field(s)'; + } + for (var att in sourceFiltering) { + if (sourceFiltering.hasOwnProperty(att) && att !== 'exclude' && att !== 'include') { + throw 'The JSON object should have only either an exclude or an include field'; + } + } + $scope.indexPattern.setSourceFiltering(sourceFiltering); + notify.info('Updated the set of retrieved fields'); + } else if ($scope.indexPattern.getSourceFiltering()) { + var confirmIfEmpty = 'The following configuration will be deleted:\n\n' + + JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); + if ($window.confirm(confirmIfEmpty)) { + $scope.indexPattern.setSourceFiltering(undefined); + notify.info('All fields are now retrieved'); + } else { + $scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); + } + } + } catch (e) { + notify.error(e); + } + }; + } + }; + }); +}); diff --git a/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js b/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js new file mode 100644 index 0000000000000..0f7e6098f37bc --- /dev/null +++ b/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js @@ -0,0 +1,50 @@ +define(function (require) { + /** + * Returns true if the path matches the given pattern + */ + function matchPath(pathPattern, path) { + var pattern = pathPattern.split('.'); + var pathArr = path.split('.'); + + if (pattern.length !== pathArr.length) { + return false; + } + for (var i = 0; i < pattern.length; i++) { + if (pattern[i] !== '*' && pattern[i] !== pathArr[i]) { + return false; + } + } + return true; + } + + function process(val, name) { + if (val.constructor === Array) { + for (var i = 0; i < val.length; i++) { + if (matchPath(val[i], name)) { + return true; + } + } + return false; + } else { + return matchPath(val, name); + } + } + + /** + * Returns true if the field named "name" should be retrieved as part of + * the _source object for each hit. + */ + return function isRetrieved(sourceFiltering, name) { + if (sourceFiltering === undefined) { + return true; + } + if (sourceFiltering.include) { + var inc = sourceFiltering.include; + return process(inc, name); + } else if (sourceFiltering.exclude) { + var exc = sourceFiltering.exclude; + return !process(exc, name); + } + return false; + }; +}); diff --git a/src/testUtils/stub_index_pattern.js b/src/testUtils/stub_index_pattern.js index d350fa1254ca6..f0ad374dbd669 100644 --- a/src/testUtils/stub_index_pattern.js +++ b/src/testUtils/stub_index_pattern.js @@ -18,6 +18,7 @@ define(function (require) { this.timeFieldName = timeField; this.getNonScriptedFields = sinon.spy(); this.getScriptedFields = sinon.spy(); + this.getSourceFiltering = sinon.spy(); this.metaFields = ['_id', '_type', '_source']; this.fieldFormatMap = {}; this.routes = IndexPattern.prototype.routes; diff --git a/src/ui/public/doc_table/__tests__/doc_table.js b/src/ui/public/doc_table/__tests__/doc_table.js index 4ff421e86c804..091b45bd87124 100644 --- a/src/ui/public/doc_table/__tests__/doc_table.js +++ b/src/ui/public/doc_table/__tests__/doc_table.js @@ -71,6 +71,10 @@ describe('docTable', function () { expect($elem.text()).to.not.be.empty(); }); + it('should set the source filtering defintion', function () { + expect($scope.indexPattern.getSourceFiltering.called).to.be(true); + }); + it('should set the indexPattern to that of the searchSource', function () { expect($scope.indexPattern).to.be(searchSource.get('index')); }); diff --git a/src/ui/public/doc_table/doc_table.js b/src/ui/public/doc_table/doc_table.js index 4bebced3c9743..85e2aca4fc4b1 100644 --- a/src/ui/public/doc_table/doc_table.js +++ b/src/ui/public/doc_table/doc_table.js @@ -81,6 +81,11 @@ define(function (require) { $scope.searchSource.size(config.get('discover:sampleSize')); $scope.searchSource.sort(getSort($scope.sorting, $scope.indexPattern)); + var sourceFiltering = $scope.indexPattern.getSourceFiltering(); + if (sourceFiltering) { + $scope.searchSource.source(sourceFiltering); + } + // Set the watcher after initialization $scope.$watchCollection('sorting', function (newSort, oldSort) { // Don't react if sort values didn't really change diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index d99f67ff607b5..e968ee44ee9d8 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -27,6 +27,7 @@ define(function (require) { timeFieldName: 'string', notExpandable: 'boolean', intervalName: 'string', + sourceFiltering: 'json', fields: 'json', fieldFormatMap: { type: 'string', @@ -122,6 +123,17 @@ define(function (require) { } }; + // Set the source filtering configuration for that index + self.setSourceFiltering = function (config) { + self.sourceFiltering = config; + self.save(); + }; + + // Get the source filtering configuration for that index + self.getSourceFiltering = function () { + return self.sourceFiltering; + }; + self.addScriptedField = function (name, script, type, lang) { type = type || 'string'; @@ -317,7 +329,8 @@ define(function (require) { edit: '/settings/indices/{{id}}', addField: '/settings/indices/{{id}}/create-field', indexedFields: '/settings/indices/{{id}}?_a=(tab:indexedFields)', - scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)' + scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)', + sourceFiltering: '/settings/indices/{{id}}?_a=(tab:sourceFiltering)' }; return IndexPattern; From f33a46fff4d5eb536865d5b867c034837771b0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Campinas?= Date: Wed, 13 Jan 2016 14:44:38 +0000 Subject: [PATCH 2/8] ditched the new 'retrieved fields' tab and added checkbox to exclude a field in the field control --- src/plugins/elasticsearch/index.js | 3 +- .../indices/__tests__/retrieved_field.js | 60 --------- .../settings/sections/indices/_edit.html | 3 +- .../public/settings/sections/indices/_edit.js | 1 - .../settings/sections/indices/_field_types.js | 3 - .../sections/indices/_indexed_fields.js | 5 +- .../sections/indices/_source_filtering.html | 114 ------------------ .../sections/indices/_source_filtering.js | 44 ------- .../sections/indices/retrieved_field.js | 50 -------- .../server/lib/init_default_field_props.js | 3 + src/ui/public/doc_table/doc_table.js | 2 +- src/ui/public/field_editor/field_editor.html | 26 ++++ src/ui/public/index_patterns/_field.js | 1 + .../public/index_patterns/_index_pattern.js | 22 ++-- .../settings/_index_pattern_create_delete.js | 1 + 15 files changed, 46 insertions(+), 292 deletions(-) delete mode 100644 src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js delete mode 100644 src/plugins/kibana/public/settings/sections/indices/_source_filtering.html delete mode 100644 src/plugins/kibana/public/settings/sections/indices/_source_filtering.js delete mode 100644 src/plugins/kibana/public/settings/sections/indices/retrieved_field.js diff --git a/src/plugins/elasticsearch/index.js b/src/plugins/elasticsearch/index.js index c2346bd01ef44..63950a737fb54 100644 --- a/src/plugins/elasticsearch/index.js +++ b/src/plugins/elasticsearch/index.js @@ -24,8 +24,7 @@ module.exports = function (kibana) { key: Joi.string() }).default(), apiVersion: Joi.string().default('2.0'), - engineVersion: Joi.string().valid('^2.1.0').default('^2.1.0'), - plugins: Joi.array() + engineVersion: Joi.string().valid('^2.1.0').default('^2.1.0') }).default(); }, diff --git a/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js b/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js deleted file mode 100644 index 0543e77901163..0000000000000 --- a/src/plugins/kibana/public/settings/sections/indices/__tests__/retrieved_field.js +++ /dev/null @@ -1,60 +0,0 @@ -var expect = require('expect.js'); - -define(function (require) { - var isRetrieved = require('src/plugins/kibana/public/settings/sections/indices/retrieved_field'); - - describe('Settings', function () { - describe('Indices', function () { - describe('isRetrieved(sourceFiltering, name)', function () { - it('should be a function', function () { - expect(isRetrieved).to.be.a(Function); - }); - - it('should retrieve john', function () { - var sourceFiltering = { - include: 'john' - }; - - expect(isRetrieved(sourceFiltering, 'john')).to.be(true); - }); - - it('should not retrieve connor', function () { - var sourceFiltering = { - exclude: 'connor' - }; - - expect(isRetrieved(sourceFiltering, 'connor')).to.be(false); - }); - - it('should retrieve connor', function () { - var sourceFiltering = { - exclude: '*.connor' - }; - - expect(isRetrieved(sourceFiltering, 'connor')).to.be(true); - expect(isRetrieved(sourceFiltering, 'john.connor')).to.be(false); - }); - - it('should not retrieve neither john nor connor', function () { - var sourceFiltering = { - exclude: [ 'john', 'connor' ] - }; - - expect(isRetrieved(sourceFiltering, 'connor')).to.be(false); - expect(isRetrieved(sourceFiltering, 'john')).to.be(false); - expect(isRetrieved(sourceFiltering, 'toto')).to.be(true); - }); - - it('should not retrieve john.*.connor', function () { - var sourceFiltering = { - exclude: 'john.*.connor' - }; - - expect(isRetrieved(sourceFiltering, 'john.j.connor')).to.be(false); - expect(isRetrieved(sourceFiltering, 'john.t.connor')).to.be(false); - expect(isRetrieved(sourceFiltering, 'john.j.watterson')).to.be(true); - }); - }); - }); - }); -}); diff --git a/src/plugins/kibana/public/settings/sections/indices/_edit.html b/src/plugins/kibana/public/settings/sections/indices/_edit.html index 0c865be5a7954..ee0e6f65a307c 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_edit.html +++ b/src/plugins/kibana/public/settings/sections/indices/_edit.html @@ -41,14 +41,13 @@
  • {{ fieldType.title }} - ({{ fieldType.count }}) + ({{ fieldType.count }})
    • - diff --git a/src/plugins/kibana/public/settings/sections/indices/_edit.js b/src/plugins/kibana/public/settings/sections/indices/_edit.js index 8b7797fd1d470..abd5ee43ea8a9 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_edit.js +++ b/src/plugins/kibana/public/settings/sections/indices/_edit.js @@ -3,7 +3,6 @@ define(function (require) { require('plugins/kibana/settings/sections/indices/_indexed_fields'); require('plugins/kibana/settings/sections/indices/_scripted_fields'); require('plugins/kibana/settings/sections/indices/_index_header'); - require('plugins/kibana/settings/sections/indices/_source_filtering'); require('ui/routes') .when('/settings/indices/:indexPatternId', { diff --git a/src/plugins/kibana/public/settings/sections/indices/_field_types.js b/src/plugins/kibana/public/settings/sections/indices/_field_types.js index 6e87c1a691250..6e4e1f284315c 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_field_types.js +++ b/src/plugins/kibana/public/settings/sections/indices/_field_types.js @@ -20,9 +20,6 @@ define(function (require) { title: 'scripted fields', index: 'scriptedFields', count: fieldCount.scripted - }, { - title: 'Retrieved Fields', - index: 'sourceFiltering' }]; }; }; diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index 4aadaf598c869..cc7eb28ce4e59 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -1,6 +1,5 @@ define(function (require) { var _ = require('lodash'); - var isRetrieved = require('./retrieved_field'); require('ui/paginated_table'); require('ui/modules').get('apps/settings') @@ -35,7 +34,6 @@ define(function (require) { // clear and destroy row scopes _.invoke(rowScopes.splice(0), '$destroy'); - var sourceFiltering = $scope.indexPattern.getSourceFiltering(); var fields = filter($scope.indexPattern.getNonScriptedFields(), $scope.fieldFilter); _.find($scope.fieldTypes, {index: 'indexedFields'}).count = fields.length; // Update the tab count @@ -64,7 +62,8 @@ define(function (require) { value: field.indexed }, { - markup: isRetrieved(sourceFiltering, field.displayName) ? yesTemplate : noTemplate + markup: field.exclude ? noTemplate : yesTemplate, + value: field.exclude }, { markup: controlsHtml, diff --git a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html deleted file mode 100644 index 8af8c30333912..0000000000000 --- a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.html +++ /dev/null @@ -1,114 +0,0 @@ -

    Retrieved fields - - -

    Retrieved Fields Help

    -     -

    - -
    -

    - Retrieved Fields Help -

    - -

    - All fields are by default retrieved and are available inside the "_source" object of each hit. Thanks to the - - source filtering API - - - of Elasticsearch, you can choose which fields are actually retrieved. The value needs to be a JSON object. -

    - -
    - Examples - -

    Exclusion of a single field

    - -
    { - "exclude": "user" -}
    - -

    Exclusion with a path pattern

    -
    { - "exclude": [ - "obj1.*", - "*.value" - ] -}
    - -

    Complete control with exclusions and inclusion

    -
    { - "exclude": "obj1.*", - "include": "obj2.*.val" -}
    -
    - -

    -By default, all fields are retrieved to populate results table. -Sometimes however some fields can be very large and seriously affect performance. This can be often the case when you have materialized one to many relationships, e.g., an entity having many nested entities. Those fields are still useful for searching or analytics but not in a result table.

    -Use this setting to decide what to include or exclude in the data retrieved from the Elasticsearch index.
    -If empty, all fields are retrieved. -

    - -
    -
    {{ sourceFiltering | json }}
    - -
    - -
    -
    diff --git a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js b/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js deleted file mode 100644 index 5026857d6cd26..0000000000000 --- a/src/plugins/kibana/public/settings/sections/indices/_source_filtering.js +++ /dev/null @@ -1,44 +0,0 @@ -define(function (require) { - require('ui/modules').get('apps/settings') - .directive('sourceFiltering', function (Notifier, $window) { - var notify = new Notifier(); - return { - restrict: 'E', - template: require('plugins/kibana/settings/sections/indices/_source_filtering.html'), - link: function ($scope) { - $scope.showHelp = false; - $scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); - $scope.save = function () { - try { - var sourceFiltering; - - if ($scope.sourceFiltering) { - sourceFiltering = JSON.parse($scope.sourceFiltering); - if (sourceFiltering.constructor !== Object) { - throw 'You must enter a JSON object with exclude/include field(s)'; - } - for (var att in sourceFiltering) { - if (sourceFiltering.hasOwnProperty(att) && att !== 'exclude' && att !== 'include') { - throw 'The JSON object should have only either an exclude or an include field'; - } - } - $scope.indexPattern.setSourceFiltering(sourceFiltering); - notify.info('Updated the set of retrieved fields'); - } else if ($scope.indexPattern.getSourceFiltering()) { - var confirmIfEmpty = 'The following configuration will be deleted:\n\n' + - JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); - if ($window.confirm(confirmIfEmpty)) { - $scope.indexPattern.setSourceFiltering(undefined); - notify.info('All fields are now retrieved'); - } else { - $scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' '); - } - } - } catch (e) { - notify.error(e); - } - }; - } - }; - }); -}); diff --git a/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js b/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js deleted file mode 100644 index 0f7e6098f37bc..0000000000000 --- a/src/plugins/kibana/public/settings/sections/indices/retrieved_field.js +++ /dev/null @@ -1,50 +0,0 @@ -define(function (require) { - /** - * Returns true if the path matches the given pattern - */ - function matchPath(pathPattern, path) { - var pattern = pathPattern.split('.'); - var pathArr = path.split('.'); - - if (pattern.length !== pathArr.length) { - return false; - } - for (var i = 0; i < pattern.length; i++) { - if (pattern[i] !== '*' && pattern[i] !== pathArr[i]) { - return false; - } - } - return true; - } - - function process(val, name) { - if (val.constructor === Array) { - for (var i = 0; i < val.length; i++) { - if (matchPath(val[i], name)) { - return true; - } - } - return false; - } else { - return matchPath(val, name); - } - } - - /** - * Returns true if the field named "name" should be retrieved as part of - * the _source object for each hit. - */ - return function isRetrieved(sourceFiltering, name) { - if (sourceFiltering === undefined) { - return true; - } - if (sourceFiltering.include) { - var inc = sourceFiltering.include; - return process(inc, name); - } else if (sourceFiltering.exclude) { - var exc = sourceFiltering.exclude; - return !process(exc, name); - } - return false; - }; -}); diff --git a/src/plugins/kibana/server/lib/init_default_field_props.js b/src/plugins/kibana/server/lib/init_default_field_props.js index 0ec669ca5ac32..b7dfdb0204390 100644 --- a/src/plugins/kibana/server/lib/init_default_field_props.js +++ b/src/plugins/kibana/server/lib/init_default_field_props.js @@ -17,6 +17,7 @@ module.exports = function initDefaultFieldProps(fields) { analyzed: true, doc_values: false, scripted: false, + exclude: false, count: 0 }); @@ -27,6 +28,7 @@ module.exports = function initDefaultFieldProps(fields) { analyzed: false, doc_values: true, scripted: false, + exclude: false, count: 0 }); } @@ -36,6 +38,7 @@ module.exports = function initDefaultFieldProps(fields) { analyzed: false, doc_values: true, scripted: false, + exclude: false, count: 0 }); } diff --git a/src/ui/public/doc_table/doc_table.js b/src/ui/public/doc_table/doc_table.js index 85e2aca4fc4b1..7344efc88b50b 100644 --- a/src/ui/public/doc_table/doc_table.js +++ b/src/ui/public/doc_table/doc_table.js @@ -81,7 +81,7 @@ define(function (require) { $scope.searchSource.size(config.get('discover:sampleSize')); $scope.searchSource.sort(getSort($scope.sorting, $scope.indexPattern)); - var sourceFiltering = $scope.indexPattern.getSourceFiltering(); + var sourceFiltering = $scope.indexPattern.getSourceFiltering($scope.columns); if (sourceFiltering) { $scope.searchSource.source(sourceFiltering); } diff --git a/src/ui/public/field_editor/field_editor.html b/src/ui/public/field_editor/field_editor.html index 18e065a6b7b6d..2293bc35ec3c7 100644 --- a/src/ui/public/field_editor/field_editor.html +++ b/src/ui/public/field_editor/field_editor.html @@ -83,6 +83,32 @@

    +
    + + Help + + + + +
    +

    + Field Exclusion +

    + +

    + If checked, this field will not be retrieved as part of the _source object. This is thanks to the + + source filtering API + + + of Elasticsearch. +

    +

    + If this field is explicitly selected in your saved search, then it will not be excluded. +

    +
    +
    +
    diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index 5352e38d89d68..a662d5beb5338 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -45,6 +45,7 @@ define(function (require) { obj.fact('name'); obj.fact('type'); obj.writ('count', spec.count || 0); + obj.writ('exclude', spec.exclude); // scripted objs obj.fact('scripted', scripted); diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index e968ee44ee9d8..a28354d9fbdbf 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -27,7 +27,6 @@ define(function (require) { timeFieldName: 'string', notExpandable: 'boolean', intervalName: 'string', - sourceFiltering: 'json', fields: 'json', fieldFormatMap: { type: 'string', @@ -123,15 +122,15 @@ define(function (require) { } }; - // Set the source filtering configuration for that index - self.setSourceFiltering = function (config) { - self.sourceFiltering = config; - self.save(); - }; - - // Get the source filtering configuration for that index - self.getSourceFiltering = function () { - return self.sourceFiltering; + // Get the source filtering configuration for that index. + // Fields which name appears in the given columns array will not be excluded. + self.getSourceFiltering = function (columns) { + return { + exclude: _(self.getNonScriptedFields()) + .filter((field) => field.exclude && !_.contains(columns, field.name)) + .map((field) => field.name) + .value() + }; }; self.addScriptedField = function (name, script, type, lang) { @@ -329,8 +328,7 @@ define(function (require) { edit: '/settings/indices/{{id}}', addField: '/settings/indices/{{id}}/create-field', indexedFields: '/settings/indices/{{id}}?_a=(tab:indexedFields)', - scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)', - sourceFiltering: '/settings/indices/{{id}}?_a=(tab:sourceFiltering)' + scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)' }; return IndexPattern; diff --git a/test/functional/apps/settings/_index_pattern_create_delete.js b/test/functional/apps/settings/_index_pattern_create_delete.js index 9c8d27b810cb3..d6911b0fb7ac6 100644 --- a/test/functional/apps/settings/_index_pattern_create_delete.js +++ b/test/functional/apps/settings/_index_pattern_create_delete.js @@ -53,6 +53,7 @@ define(function (require) { 'format', 'analyzed', 'indexed', + 'retrieved', 'controls' ]; From 618bb9ba112558034348172d74d29dd6e60a11ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Campinas?= Date: Fri, 15 Jan 2016 15:39:47 +0000 Subject: [PATCH 3/8] disable field exclusion checkbox if field is a metafield --- .../settings/sections/indices/_indexed_fields.js | 8 +++++--- src/ui/public/field_editor/field_editor.html | 12 +++++++++++- src/ui/public/field_editor/field_editor.js | 5 ++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index cc7eb28ce4e59..341506937e6b7 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -3,7 +3,7 @@ define(function (require) { require('ui/paginated_table'); require('ui/modules').get('apps/settings') - .directive('indexedFields', function ($filter) { + .directive('indexedFields', function ($filter, config) { var yesTemplate = ''; var noTemplate = ''; var nameHtml = require('plugins/kibana/settings/sections/indices/_field_name.html'); @@ -34,11 +34,13 @@ define(function (require) { // clear and destroy row scopes _.invoke(rowScopes.splice(0), '$destroy'); + const metaFields = config.get('metaFields'); var fields = filter($scope.indexPattern.getNonScriptedFields(), $scope.fieldFilter); _.find($scope.fieldTypes, {index: 'indexedFields'}).count = fields.length; // Update the tab count $scope.rows = fields.map(function (field) { var childScope = _.assign($scope.$new(), { field: field }); + const isMetaField = _.contains(metaFields, field.name); rowScopes.push(childScope); return [ @@ -62,8 +64,8 @@ define(function (require) { value: field.indexed }, { - markup: field.exclude ? noTemplate : yesTemplate, - value: field.exclude + markup: isMetaField || !!field.exclude ? noTemplate : yesTemplate, + value: isMetaField || !!field.exclude }, { markup: controlsHtml, diff --git a/src/ui/public/field_editor/field_editor.html b/src/ui/public/field_editor/field_editor.html index 2293bc35ec3c7..7ee38f13ee896 100644 --- a/src/ui/public/field_editor/field_editor.html +++ b/src/ui/public/field_editor/field_editor.html @@ -88,7 +88,17 @@

    Help - + + +
    +

    + Field Exclusion +

    + +

    + You cannot exclude metadata fields. +

    +

    diff --git a/src/ui/public/field_editor/field_editor.js b/src/ui/public/field_editor/field_editor.js index f62ea74ef1e0a..d59f2aefc2838 100644 --- a/src/ui/public/field_editor/field_editor.js +++ b/src/ui/public/field_editor/field_editor.js @@ -21,16 +21,19 @@ define(function (require) { getField: '&field' }, controllerAs: 'editor', - controller: function ($scope, Notifier, kbnUrl) { + controller: function ($scope, Notifier, kbnUrl, config) { var self = this; var notify = new Notifier({ location: 'Field Editor' }); + const metaFields = config.get('metaFields'); + self.scriptingInfo = scriptingInfo; self.scriptingWarning = scriptingWarning; self.indexPattern = $scope.getIndexPattern(); self.field = shadowCopy($scope.getField()); self.formatParams = self.field.format.params(); + $scope.isMetaField = _.contains(metaFields, self.field.name); // only init on first create self.creating = !self.indexPattern.fields.byName[self.field.name]; From a612140865c9620f5fd368f61bd25fa3d9d8995e Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 21 Jan 2016 09:01:48 -0700 Subject: [PATCH 4/8] [indexPattern] copy excluded field property when refreshing fields --- src/ui/public/index_patterns/_index_pattern.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js index a28354d9fbdbf..861f2a12a7f99 100644 --- a/src/ui/public/index_patterns/_index_pattern.js +++ b/src/ui/public/index_patterns/_index_pattern.js @@ -293,8 +293,19 @@ define(function (require) { }; self._fetchFields = function () { + var existingFieldsByName = self.fields.byName; + return mapper.getFieldsForIndexPattern(self, true) .then(function (fields) { + + // copy over kibana-added properties from existing fields + fields.forEach(function (field) { + var existingField = existingFieldsByName[field.name]; + if (existingField) { + field.exclude = existingField.exclude; + } + }); + // append existing scripted fields fields = fields.concat(self.getScriptedFields()); From 24977bcd4585b7d5d30f206faf90571cbdd13801 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 21 Jan 2016 09:22:52 -0700 Subject: [PATCH 5/8] [indexPattern/field] move isMetaField consideration into Field --- .../public/settings/sections/indices/_indexed_fields.js | 6 ++---- src/ui/public/index_patterns/_field.js | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index 341506937e6b7..04d6fa8e2c4aa 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -34,13 +34,11 @@ define(function (require) { // clear and destroy row scopes _.invoke(rowScopes.splice(0), '$destroy'); - const metaFields = config.get('metaFields'); var fields = filter($scope.indexPattern.getNonScriptedFields(), $scope.fieldFilter); _.find($scope.fieldTypes, {index: 'indexedFields'}).count = fields.length; // Update the tab count $scope.rows = fields.map(function (field) { var childScope = _.assign($scope.$new(), { field: field }); - const isMetaField = _.contains(metaFields, field.name); rowScopes.push(childScope); return [ @@ -64,8 +62,8 @@ define(function (require) { value: field.indexed }, { - markup: isMetaField || !!field.exclude ? noTemplate : yesTemplate, - value: isMetaField || !!field.exclude + markup: !field.exclude ? yesTemplate : noTemplate, + value: !field.exclude }, { markup: controlsHtml, diff --git a/src/ui/public/index_patterns/_field.js b/src/ui/public/index_patterns/_field.js index a662d5beb5338..c7ad7724fade2 100644 --- a/src/ui/public/index_patterns/_field.js +++ b/src/ui/public/index_patterns/_field.js @@ -1,5 +1,5 @@ define(function (require) { - return function FieldObjectProvider(Private, shortDotsFilter, $rootScope, Notifier) { + return function FieldObjectProvider(Private, shortDotsFilter, $rootScope, Notifier, config) { var notify = new Notifier({ location: 'IndexPattern Field' }); var FieldFormat = Private(require('ui/index_patterns/_field_format/FieldFormat')); var fieldTypes = Private(require('ui/index_patterns/_field_types')); @@ -41,11 +41,12 @@ define(function (require) { var sortable = spec.name === '_score' || ((indexed || scripted) && type.sortable); var bucketable = indexed || scripted; var filterable = spec.name === '_id' || scripted || (indexed && type.filterable); + var isMetaField = config.get('metaFields').includes(spec.name); obj.fact('name'); obj.fact('type'); obj.writ('count', spec.count || 0); - obj.writ('exclude', spec.exclude); + obj.fact('exclude', Boolean(!isMetaField && spec.exclude)); // scripted objs obj.fact('scripted', scripted); From 6d8b451f8f3ff6d7e8038226fcdb7b2cab64a917 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 21 Jan 2016 09:24:44 -0700 Subject: [PATCH 6/8] [indexPattern/edit] invert the "retreived" column, for accuracy --- .../public/settings/sections/indices/_indexed_fields.js | 6 +++--- .../apps/settings/_index_pattern_create_delete.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index 04d6fa8e2c4aa..980999e45e676 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -24,7 +24,7 @@ define(function (require) { { title: 'format' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, - { title: 'retrieved', info: 'Fields that are not retrieved as part of the _source object per hit' }, + { title: 'exclude', info: 'Fields that are not excluded from _source when _source is fetched' }, { title: 'controls', sortable: false } ]; @@ -62,8 +62,8 @@ define(function (require) { value: field.indexed }, { - markup: !field.exclude ? yesTemplate : noTemplate, - value: !field.exclude + markup: field.exclude ? yesTemplate : noTemplate, + value: field.exclude }, { markup: controlsHtml, diff --git a/test/functional/apps/settings/_index_pattern_create_delete.js b/test/functional/apps/settings/_index_pattern_create_delete.js index d6911b0fb7ac6..a86444b2798a9 100644 --- a/test/functional/apps/settings/_index_pattern_create_delete.js +++ b/test/functional/apps/settings/_index_pattern_create_delete.js @@ -53,7 +53,7 @@ define(function (require) { 'format', 'analyzed', 'indexed', - 'retrieved', + 'exclude', 'controls' ]; From fb550a3142b077ba73a9aa4bf46a456e2a9f87ca Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 21 Jan 2016 09:25:24 -0700 Subject: [PATCH 7/8] [indexPattern/field] touchup the field.exclude message --- src/ui/public/field_editor/field_editor.html | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ui/public/field_editor/field_editor.html b/src/ui/public/field_editor/field_editor.html index 7ee38f13ee896..d7b74a224fba9 100644 --- a/src/ui/public/field_editor/field_editor.html +++ b/src/ui/public/field_editor/field_editor.html @@ -106,12 +106,8 @@

    - If checked, this field will not be retrieved as part of the _source object. This is thanks to the - - source filtering API - - - of Elasticsearch. + If checked, this field will be filtered from the _source of each document using + source filtering. This only impacts views that fetch the _source for documents, like Discover or the doc table.

    If this field is explicitly selected in your saved search, then it will not be excluded. From 5fa0df9cc13a07bb24fca0862c5c9f2d4a3f585b Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 21 Jan 2016 12:33:14 -0700 Subject: [PATCH 8/8] Fix typo --- .../kibana/public/settings/sections/indices/_indexed_fields.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js index 980999e45e676..a60237aaba06c 100644 --- a/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js +++ b/src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js @@ -24,7 +24,7 @@ define(function (require) { { title: 'format' }, { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' }, { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' }, - { title: 'exclude', info: 'Fields that are not excluded from _source when _source is fetched' }, + { title: 'exclude', info: 'Fields that are excluded from _source when it is fetched' }, { title: 'controls', sortable: false } ];