diff --git a/src/core_plugins/kibana/public/management/sections/indices/source_filters/filter.html b/src/core_plugins/kibana/public/management/sections/indices/source_filters/filter.html
new file mode 100644
index 0000000000000..e4817c9920513
--- /dev/null
+++ b/src/core_plugins/kibana/public/management/sections/indices/source_filters/filter.html
@@ -0,0 +1,12 @@
+
+ {{ filter.value }}
+
+
+
diff --git a/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.html b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.html
new file mode 100644
index 0000000000000..f75f1c4b6a1c7
--- /dev/null
+++ b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.html
@@ -0,0 +1,37 @@
+
Source Filters
+
+
+ Source filters can be used to exclude one or more fields when fetching the document source. This happens when viewing a document in the Discover app, or with a table displaying results from a saved search in the Dashboard app. Each row is built using the source of a single document, and if you have documents with large or unimportant fields you may benefit from filtering those out at this lower level.
+
+
+
+ Note that multi-fields will incorrectly appear as matches in the table below. These filters only actually apply to fields in the original source document, so matching multi-fields are not actually being filtered.
+
+
+
+
+
+
+
+
+
diff --git a/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.js b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.js
new file mode 100644
index 0000000000000..adcb4af477f8b
--- /dev/null
+++ b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.js
@@ -0,0 +1,119 @@
+import { find, each, escape, invoke, size, without } from 'lodash';
+
+import uiModules from 'ui/modules';
+import Notifier from 'ui/notify/notifier';
+import FieldWildcardProvider from 'ui/field_wildcard';
+
+import controlsHtml from 'plugins/kibana/management/sections/indices/source_filters/controls.html';
+import filterHtml from 'plugins/kibana/management/sections/indices/source_filters/filter.html';
+import template from './source_filters.html';
+import './source_filters.less';
+
+const notify = new Notifier();
+
+uiModules.get('kibana')
+.directive('sourceFilters', function (Private, $filter) {
+ const angularFilter = $filter('filter');
+ const { fieldWildcardMatcher } = Private(FieldWildcardProvider);
+ const rowScopes = []; // track row scopes, so they can be destroyed as needed
+
+ return {
+ restrict: 'E',
+ scope: {
+ indexPattern: '='
+ },
+ template,
+ controllerAs: 'sourceFilters',
+ controller: class FieldFiltersController {
+ constructor($scope) {
+ if (!$scope.indexPattern) {
+ throw new Error('index pattern is required');
+ }
+
+ $scope.perPage = 25;
+ $scope.columns = [
+ {
+ title: 'filter'
+ },
+ {
+ title: 'matches',
+ sortable: false,
+ info: 'The source fields that match the filter.'
+ },
+ {
+ title: 'controls',
+ sortable: false
+ }
+ ];
+
+ this.$scope = $scope;
+ this.saving = false;
+ this.editing = null;
+ this.newValue = null;
+ this.placeHolder = 'source filter, accepts wildcards (e.g., `user*` to filter fields starting with \'user\')';
+
+ $scope.$watchMulti([ '[]indexPattern.sourceFilters', '$parent.fieldFilter' ], () => {
+ invoke(rowScopes, '$destroy');
+ rowScopes.length = 0;
+
+ if ($scope.indexPattern.sourceFilters) {
+ $scope.rows = [];
+ each($scope.indexPattern.sourceFilters, (filter) => {
+ const matcher = fieldWildcardMatcher([ filter.value ]);
+ // compute which fields match a filter
+ const matches = $scope.indexPattern.getNonScriptedFields().map(f => f.name).filter(matcher).sort();
+ if ($scope.$parent.fieldFilter && !angularFilter(matches, $scope.$parent.fieldFilter).length) {
+ return;
+ }
+ // compute the rows
+ const rowScope = $scope.$new();
+ rowScope.filter = filter;
+ rowScopes.push(rowScope);
+ $scope.rows.push([
+ {
+ markup: filterHtml,
+ scope: rowScope
+ },
+ size(matches) ? escape(matches.join(', ')) : 'The source filter doesn\'t match any known fields.',
+ {
+ markup: controlsHtml,
+ scope: rowScope
+ }
+ ]);
+ });
+ // Update the tab count
+ find($scope.$parent.editSections, {index: 'sourceFilters'}).count = $scope.rows.length;
+ }
+ });
+ }
+
+ all() {
+ return this.$scope.indexPattern.sourceFilters || [];
+ }
+
+ delete(filter) {
+ if (this.editing === filter) {
+ this.editing = null;
+ }
+
+ this.$scope.indexPattern.sourceFilters = without(this.all(), filter);
+ return this.save();
+ }
+
+ create() {
+ const value = this.newValue;
+ this.newValue = null;
+ this.$scope.indexPattern.sourceFilters = [...this.all(), { value }];
+ return this.save();
+ }
+
+ save() {
+ this.saving = true;
+ this.$scope.indexPattern.save()
+ .then(() => this.editing = null)
+ .catch(notify.error)
+ .finally(() => this.saving = false);
+ }
+ }
+ };
+});
diff --git a/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.less b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.less
new file mode 100644
index 0000000000000..b0a4cc9fcb5e3
--- /dev/null
+++ b/src/core_plugins/kibana/public/management/sections/indices/source_filters/source_filters.less
@@ -0,0 +1,34 @@
+@import (reference) "~ui/styles/variables";
+
+source-filters {
+ .header {
+ text-align: center;
+ }
+
+ .source-filters-container {
+ margin-top: 15px;
+
+ &.saving {
+ pointer-events: none;
+ opacity: .4;
+ transition: opacity 0.75s;
+ }
+
+ .source-filter {
+ display: flex;
+ align-items: center;
+ margin: 10px 0;
+
+ .value {
+ text-align: left;
+ flex: 1 1 auto;
+ padding-right: 5px;
+ font-family: @font-family-monospace;
+
+ :not(input) {
+ padding-left: 15px;
+ }
+ }
+ }
+ }
+}
diff --git a/src/core_plugins/kibana/server/lib/schemas/resources/index_pattern_schema.js b/src/core_plugins/kibana/server/lib/schemas/resources/index_pattern_schema.js
index 76cf5e111726c..0465aa411c43f 100644
--- a/src/core_plugins/kibana/server/lib/schemas/resources/index_pattern_schema.js
+++ b/src/core_plugins/kibana/server/lib/schemas/resources/index_pattern_schema.js
@@ -6,6 +6,7 @@ module.exports = Joi.object({
time_field_name: Joi.string(),
interval_name: Joi.string(),
not_expandable: Joi.boolean(),
+ source_filters: Joi.array(),
fields: Joi.array().items(
Joi.object({
name: Joi.string().required(),
diff --git a/src/test_utils/stub_index_pattern.js b/src/test_utils/stub_index_pattern.js
index 84f33eb229438..c3be4bcbd87e8 100644
--- a/src/test_utils/stub_index_pattern.js
+++ b/src/test_utils/stub_index_pattern.js
@@ -20,6 +20,7 @@ export default function (Private) {
this.timeFieldName = timeField;
this.getNonScriptedFields = sinon.spy();
this.getScriptedFields = sinon.spy();
+ this.getSourceFiltering = sinon.spy();
this.metaFields = ['_id', '_type', '_source'];
this.fieldFormatMap = {};
this.routes = IndexPattern.routes;
diff --git a/src/ui/public/courier/data_source/__tests__/search_source.js b/src/ui/public/courier/data_source/__tests__/search_source.js
index 7559ef9e88eba..4137557a40f31 100644
--- a/src/ui/public/courier/data_source/__tests__/search_source.js
+++ b/src/ui/public/courier/data_source/__tests__/search_source.js
@@ -4,17 +4,25 @@ import sinon from 'auto-release-sinon';
import RequestQueueProv from '../../_request_queue';
import SearchSourceProv from '../search_source';
+import StubIndexPatternProv from 'test_utils/stub_index_pattern';
describe('SearchSource', function () {
require('test_utils/no_digest_promises').activateForSuite();
let requestQueue;
let SearchSource;
+ let indexPattern;
+ let indexPattern2;
beforeEach(ngMock.module('kibana'));
beforeEach(ngMock.inject(function (Private) {
requestQueue = Private(RequestQueueProv);
SearchSource = Private(SearchSourceProv);
+
+ const IndexPattern = Private(StubIndexPatternProv);
+ indexPattern = new IndexPattern('test-*', null, []);
+ indexPattern2 = new IndexPattern('test2-*', null, []);
+ expect(indexPattern).to.not.be(indexPattern2);
}));
describe('#onResults()', function () {
@@ -56,4 +64,91 @@ describe('SearchSource', function () {
expect(requestQueue).to.have.length(0);
});
});
+
+ describe('#index()', function () {
+ describe('auto-sourceFiltering', function () {
+ context('new index pattern assigned', function () {
+ it('generates a source filter', function () {
+ const source = new SearchSource();
+ expect(source.get('index')).to.be(undefined);
+ expect(source.get('source')).to.be(undefined);
+ source.set('index', indexPattern);
+ expect(source.get('index')).to.be(indexPattern);
+ expect(source.get('source')).to.be.a('function');
+ });
+
+ it('removes created source filter on removal', function () {
+ const source = new SearchSource();
+ source.set('index', indexPattern);
+ source.set('index', null);
+ expect(source.get('index')).to.be(undefined);
+ expect(source.get('source')).to.be(undefined);
+ });
+ });
+
+ context('new index pattern assigned over another', function () {
+ it('replaces source filter with new', function () {
+ const source = new SearchSource();
+ source.set('index', indexPattern);
+ const sourceFilter1 = source.get('source');
+ source.set('index', indexPattern2);
+ expect(source.get('index')).to.be(indexPattern2);
+ expect(source.get('source')).to.be.a('function');
+ expect(source.get('source')).to.not.be(sourceFilter1);
+ });
+
+ it('removes created source filter on removal', function () {
+ const source = new SearchSource();
+ source.set('index', indexPattern);
+ source.set('index', indexPattern2);
+ source.set('index', null);
+ expect(source.get('index')).to.be(undefined);
+ expect(source.get('source')).to.be(undefined);
+ });
+ });
+
+ context('ip assigned before custom source filter', function () {
+ it('custom source filter becomes new source', function () {
+ const source = new SearchSource();
+ const football = {};
+ source.set('index', indexPattern);
+ expect(source.get('source')).to.be.a('function');
+ source.set('source', football);
+ expect(source.get('index')).to.be(indexPattern);
+ expect(source.get('source')).to.be(football);
+ });
+
+ it('custom source stays after removal', function () {
+ const source = new SearchSource();
+ const football = {};
+ source.set('index', indexPattern);
+ source.set('source', football);
+ source.set('index', null);
+ expect(source.get('index')).to.be(undefined);
+ expect(source.get('source')).to.be(football);
+ });
+ });
+
+ context('ip assigned after custom source filter', function () {
+ it('leaves the custom filter in place', function () {
+ const source = new SearchSource();
+ const football = {};
+ source.set('source', football);
+ source.set('index', indexPattern);
+ expect(source.get('index')).to.be(indexPattern);
+ expect(source.get('source')).to.be(football);
+ });
+
+ it('custom source stays after removal', function () {
+ const source = new SearchSource();
+ const football = {};
+ source.set('source', football);
+ source.set('index', indexPattern);
+ source.set('index', null);
+ expect(source.get('index')).to.be(undefined);
+ expect(source.get('source')).to.be(football);
+ });
+ });
+ });
+ });
});
diff --git a/src/ui/public/courier/data_source/_abstract.js b/src/ui/public/courier/data_source/_abstract.js
index 4829ef661944a..0b6832f876ebe 100644
--- a/src/ui/public/courier/data_source/_abstract.js
+++ b/src/ui/public/courier/data_source/_abstract.js
@@ -7,11 +7,13 @@ import RequestQueueProvider from '../_request_queue';
import ErrorHandlersProvider from '../_error_handlers';
import FetchProvider from '../fetch';
import DecorateQueryProvider from './_decorate_query';
+import FieldWildcardProvider from '../../field_wildcard';
export default function SourceAbstractFactory(Private, Promise, PromiseEmitter) {
let requestQueue = Private(RequestQueueProvider);
let errorHandlers = Private(ErrorHandlersProvider);
let courierFetch = Private(FetchProvider);
+ let { fieldWildcardFilter } = Private(FieldWildcardProvider);
function SourceAbstract(initialState, strategy) {
let self = this;
@@ -285,12 +287,17 @@ export default function SourceAbstractFactory(Private, Promise, PromiseEmitter)
if (flatState.body.size > 0) {
let computedFields = flatState.index.getComputedFields();
flatState.body.stored_fields = computedFields.storedFields;
- flatState.body._source = computedFields._source;
flatState.body.script_fields = flatState.body.script_fields || {};
flatState.body.docvalue_fields = flatState.body.docvalue_fields || [];
_.extend(flatState.body.script_fields, computedFields.scriptFields);
flatState.body.docvalue_fields = _.union(flatState.body.docvalue_fields, computedFields.docvalueFields);
+
+ if (flatState.body._source) {
+ // exclude source fields for this index pattern specified by the user
+ const filter = fieldWildcardFilter(flatState.body._source.excludes);
+ flatState.body.docvalue_fields = flatState.body.docvalue_fields.filter(filter);
+ }
}
decorateQuery(flatState.body.query);
diff --git a/src/ui/public/courier/data_source/search_source.js b/src/ui/public/courier/data_source/search_source.js
index 109cca094ca55..f4279aee4940d 100644
--- a/src/ui/public/courier/data_source/search_source.js
+++ b/src/ui/public/courier/data_source/search_source.js
@@ -66,6 +66,12 @@ export default function SearchSourceFactory(Promise, Private, config) {
let searchStrategy = Private(SearchStrategyProvider);
let normalizeSortRequest = Private(NormalizeSortRequestProvider);
+ let forIp = Symbol('for which index pattern?');
+
+ function isIndexPattern(val) {
+ return Boolean(val && typeof val.toIndexList === 'function');
+ }
+
_.class(SearchSource).inherits(SourceAbstract);
function SearchSource(initialState) {
SearchSource.Super.call(this, initialState, searchStrategy);
@@ -94,13 +100,31 @@ export default function SearchSourceFactory(Promise, Private, config) {
];
SearchSource.prototype.index = function (indexPattern) {
- if (indexPattern === undefined) return this._state.index;
- if (indexPattern === null) return delete this._state.index;
- if (!indexPattern || typeof indexPattern.toIndexList !== 'function') {
+ let state = this._state;
+
+ let hasSource = state.source;
+ let sourceCameFromIp = hasSource && state.source.hasOwnProperty(forIp);
+ let sourceIsForOurIp = sourceCameFromIp && state.source[forIp] === state.index;
+ if (sourceIsForOurIp) {
+ delete state.source;
+ }
+
+ if (indexPattern === undefined) return state.index;
+ if (indexPattern === null) return delete state.index;
+ if (!isIndexPattern(indexPattern)) {
throw new TypeError('expected indexPattern to be an IndexPattern duck.');
}
- this._state.index = indexPattern;
+ state.index = indexPattern;
+ if (!state.source) {
+ // imply source filtering based on the index pattern, but allow overriding
+ // it by simply setting another value for "source". When index is changed
+ state.source = function () {
+ return indexPattern.getSourceFiltering();
+ };
+ state.source[forIp] = indexPattern;
+ }
+
return this;
};
diff --git a/src/ui/public/field_wildcard/__tests__/field_wildcard.js b/src/ui/public/field_wildcard/__tests__/field_wildcard.js
new file mode 100644
index 0000000000000..9a3af53077d93
--- /dev/null
+++ b/src/ui/public/field_wildcard/__tests__/field_wildcard.js
@@ -0,0 +1,102 @@
+import expect from 'expect.js';
+import ngMock from 'ng_mock';
+
+import FieldWildcardProvider from '../../field_wildcard';
+
+describe('fieldWildcard', function () {
+ let fieldWildcardFilter;
+ let makeRegEx;
+
+ beforeEach(ngMock.module('kibana'));
+ beforeEach(ngMock.inject(function (config, Private) {
+ config.set('metaFields', ['_id', '_type', '_source']);
+ const fieldWildcard = Private(FieldWildcardProvider);
+ fieldWildcardFilter = fieldWildcard.fieldWildcardFilter;
+ makeRegEx = fieldWildcard.makeRegEx;
+ }));
+
+ describe('makeRegEx', function () {
+ it('matches * in any position', function () {
+ expect('aaaaaabbbbbbbcccccc').to.match(makeRegEx('*a*b*c*'));
+ expect('a1234').to.match(makeRegEx('*1234'));
+ expect('1234a').to.match(makeRegEx('1234*'));
+ expect('12a34').to.match(makeRegEx('12a34'));
+ });
+
+ it('properly escapes regexp control characters', function () {
+ expect('account[user_id]').to.match(makeRegEx('account[*]'));
+ });
+
+ it('properly limits matches without wildcards', function () {
+ expect('username').to.match(makeRegEx('*name'));
+ expect('username').to.match(makeRegEx('user*'));
+ expect('username').to.match(makeRegEx('username'));
+ expect('username').to.not.match(makeRegEx('user'));
+ expect('username').to.not.match(makeRegEx('name'));
+ expect('username').to.not.match(makeRegEx('erna'));
+ });
+ });
+
+ describe('filter', function () {
+ it('filters nothing when given an empty array', function () {
+ const filter = fieldWildcardFilter([]);
+ const original = [
+ 'foo',
+ 'bar',
+ 'baz',
+ 1234
+ ];
+
+ expect(original.filter(filter)).to.eql(original);
+ });
+
+ it('does not filter metaFields', function () {
+ const filter = fieldWildcardFilter([ '_*' ]);
+
+ const original = [
+ '_id',
+ '_type',
+ '_typefake'
+ ];
+
+ expect(original.filter(filter)).to.eql(['_id', '_type']);
+ });
+
+ it('filters values that match the globs', function () {
+ const filter = fieldWildcardFilter([
+ 'f*',
+ '*4'
+ ]);
+
+ const original = [
+ 'foo',
+ 'bar',
+ 'baz',
+ 1234
+ ];
+
+ expect(original.filter(filter)).to.eql(['bar', 'baz']);
+ });
+
+ it('handles weird values okay', function () {
+ const filter = fieldWildcardFilter([
+ 'f*',
+ '*4',
+ 'undefined'
+ ]);
+
+ const original = [
+ 'foo',
+ null,
+ 'bar',
+ undefined,
+ {},
+ [],
+ 'baz',
+ 1234
+ ];
+
+ expect(original.filter(filter)).to.eql([null, 'bar', {}, [], 'baz']);
+ });
+ });
+});
diff --git a/src/ui/public/field_wildcard/field_wildcard.js b/src/ui/public/field_wildcard/field_wildcard.js
new file mode 100644
index 0000000000000..ee3ca719a4d2b
--- /dev/null
+++ b/src/ui/public/field_wildcard/field_wildcard.js
@@ -0,0 +1,28 @@
+import { endsWith, escapeRegExp, memoize } from 'lodash';
+
+export default function fieldWildcard(config) {
+ const metaFields = config.get('metaFields');
+
+ const makeRegEx = memoize(function makeRegEx(glob) {
+ return new RegExp('^' + glob.split('*').map(escapeRegExp).join('.*') + '$');
+ });
+
+ function fieldWildcardMatcher(globs) {
+ return function matcher(val) {
+ // do not test metaFields or keyword
+ if (metaFields.indexOf(val) !== -1) {
+ return false;
+ }
+ return globs.some(p => makeRegEx(p).test(val));
+ };
+ }
+
+ function fieldWildcardFilter(globs) {
+ const matcher = fieldWildcardMatcher(globs);
+ return function filter(val) {
+ return !matcher(val);
+ };
+ }
+
+ return { makeRegEx, fieldWildcardMatcher, fieldWildcardFilter };
+};
diff --git a/src/ui/public/index_patterns/__tests__/_get_computed_fields.js b/src/ui/public/index_patterns/__tests__/_get_computed_fields.js
index ec7e31ae56c94..7f38326d9117c 100644
--- a/src/ui/public/index_patterns/__tests__/_get_computed_fields.js
+++ b/src/ui/public/index_patterns/__tests__/_get_computed_fields.js
@@ -26,10 +26,6 @@ describe('get computed fields', function () {
expect(fn().storedFields).to.contain('*');
});
- it('should request _source seperately', function () {
- expect(fn()._source).to.be(true);
- });
-
it('should request date fields as docvalue_fields', function () {
expect(fn().docvalueFields).to.contain('@timestamp');
expect(fn().docvalueFields).to.not.contain('bytes');
diff --git a/src/ui/public/index_patterns/_get_computed_fields.js b/src/ui/public/index_patterns/_get_computed_fields.js
index 50f56eda4c47d..3038c5e55468e 100644
--- a/src/ui/public/index_patterns/_get_computed_fields.js
+++ b/src/ui/public/index_patterns/_get_computed_fields.js
@@ -19,7 +19,6 @@ export default function () {
return {
storedFields: ['*'],
- _source: true,
scriptFields: scriptFields,
docvalueFields: docvalueFields
};
diff --git a/src/ui/public/index_patterns/_index_pattern.js b/src/ui/public/index_patterns/_index_pattern.js
index 6a088619ec407..d0bb4a21cd5d6 100644
--- a/src/ui/public/index_patterns/_index_pattern.js
+++ b/src/ui/public/index_patterns/_index_pattern.js
@@ -33,7 +33,8 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
edit: '/management/kibana/indices/{{id}}',
addField: '/management/kibana/indices/{{id}}/create-field',
indexedFields: '/management/kibana/indices/{{id}}?_a=(tab:indexedFields)',
- scriptedFields: '/management/kibana/indices/{{id}}?_a=(tab:scriptedFields)'
+ scriptedFields: '/management/kibana/indices/{{id}}?_a=(tab:scriptedFields)',
+ sourceFilters: '/management/kibana/indices/{{id}}?_a=(tab:sourceFilters)'
});
const mapping = mappingSetup.expandShorthand({
@@ -42,6 +43,7 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
notExpandable: 'boolean',
intervalName: 'string',
fields: 'json',
+ sourceFilters: 'json',
fieldFormatMap: {
type: 'string',
_serialize(map = {}) {
@@ -198,6 +200,13 @@ export default function IndexPatternFactory(Private, Notifier, config, kbnIndex,
.then(() => this);
}
+ // Get the source filtering configuration for that index.
+ getSourceFiltering() {
+ return {
+ excludes: this.sourceFilters && this.sourceFilters.map(filter => filter.value) || []
+ };
+ }
+
addScriptedField(name, script, type = 'string', lang) {
const scriptedFields = this.getScriptedFields();
const names = _.pluck(scriptedFields, 'name');
diff --git a/src/ui/public/watch_multi/watch_multi.js b/src/ui/public/watch_multi/watch_multi.js
index 279076eea4e8b..42a3fd30fe978 100644
--- a/src/ui/public/watch_multi/watch_multi.js
+++ b/src/ui/public/watch_multi/watch_multi.js
@@ -25,7 +25,7 @@ uiModules.get('kibana')
* 4. a function that will be called, like a normal function water
*
* 5. an object with any of the properties:
- * `get`: the getter called on each itteration
+ * `get`: the getter called on each iteration
* `deep`: a flag to turn on objectEquality in $watch
* `fn`: the watch registration function ($scope.$watch or $scope.$watchCollection)
*
diff --git a/test/fixtures/dump_data/dashboard.mapping.json b/test/fixtures/dump_data/dashboard.mapping.json
index ff55e2b8221c4..26e8fe97238a6 100644
--- a/test/fixtures/dump_data/dashboard.mapping.json
+++ b/test/fixtures/dump_data/dashboard.mapping.json
@@ -1 +1 @@
-{".kibana":{"mappings":{"config":{"properties":{"buildNum":{"type":"keyword"}}},"index-pattern":{"properties":{"fields":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timeFieldName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"search":{"properties":{"columns":{"type":"text"},"description":{"type":"text"},"hits":{"type":"integer"},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text"}}},"sort":{"type":"text"},"title":{"type":"text"},"version":{"type":"integer"}}},"visualization":{"properties":{"description":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"uiStateJSON":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"version":{"type":"integer"},"visState":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"server":{"properties":{"uuid":{"type":"keyword"}}},"dashboard":{"properties":{"description":{"type":"text"},"hits":{"type":"integer"},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text"}}},"optionsJSON":{"type":"text"},"panelsJSON":{"type":"text"},"timeFrom":{"type":"text"},"timeRestore":{"type":"boolean"},"timeTo":{"type":"text"},"title":{"type":"text"},"uiStateJSON":{"type":"text"},"version":{"type":"integer"}}}}}}
+{".kibana":{"mappings":{"config":{"properties":{"buildNum":{"type":"keyword"}}},"index-pattern":{"properties":{"fields":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timeFieldName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"sourceFilters":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"search":{"properties":{"columns":{"type":"text"},"description":{"type":"text"},"hits":{"type":"integer"},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text"}}},"sort":{"type":"text"},"title":{"type":"text"},"version":{"type":"integer"}}},"visualization":{"properties":{"description":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"uiStateJSON":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"version":{"type":"integer"},"visState":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},"server":{"properties":{"uuid":{"type":"keyword"}}},"dashboard":{"properties":{"description":{"type":"text"},"hits":{"type":"integer"},"kibanaSavedObjectMeta":{"properties":{"searchSourceJSON":{"type":"text"}}},"optionsJSON":{"type":"text"},"panelsJSON":{"type":"text"},"timeFrom":{"type":"text"},"timeRestore":{"type":"boolean"},"timeTo":{"type":"text"},"title":{"type":"text"},"uiStateJSON":{"type":"text"},"version":{"type":"integer"}}}}}}
diff --git a/test/fixtures/dump_data/visualize.mapping.json b/test/fixtures/dump_data/visualize.mapping.json
index 8c04223d702eb..f33dbe1faa201 100644
--- a/test/fixtures/dump_data/visualize.mapping.json
+++ b/test/fixtures/dump_data/visualize.mapping.json
@@ -1 +1 @@
-{".kibana":{"mappings":{"index-pattern":{"properties":{"fields":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timeFieldName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}}}
+{".kibana":{"mappings":{"index-pattern":{"properties":{"fields":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timeFieldName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"sourceFilters":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}}}
diff --git a/test/fixtures/dump_data/visualize_source-filters.data.json b/test/fixtures/dump_data/visualize_source-filters.data.json
new file mode 100644
index 0000000000000..2130cb8c3a755
--- /dev/null
+++ b/test/fixtures/dump_data/visualize_source-filters.data.json
@@ -0,0 +1 @@
+{"_index":".kibana","_type":"index-pattern","_id":"logstash-*","_score":1,"_source":{"title":"logstash-*","timeFieldName":"@timestamp","fields":"[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]","sourceFilters":"[{\"value\":\"referer\"},{\"value\":\"relatedContent*\"}]"}}
diff --git a/test/fixtures/dump_data/visualize_source-filters.mapping.json b/test/fixtures/dump_data/visualize_source-filters.mapping.json
new file mode 100644
index 0000000000000..f33dbe1faa201
--- /dev/null
+++ b/test/fixtures/dump_data/visualize_source-filters.mapping.json
@@ -0,0 +1 @@
+{".kibana":{"mappings":{"index-pattern":{"properties":{"fields":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timeFieldName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"sourceFilters":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}}}
diff --git a/test/functional/apps/discover/_source_filters.js b/test/functional/apps/discover/_source_filters.js
new file mode 100644
index 0000000000000..563d638ceb06f
--- /dev/null
+++ b/test/functional/apps/discover/_source_filters.js
@@ -0,0 +1,51 @@
+import {
+ bdd,
+ scenarioManager,
+ esClient,
+ elasticDump
+} from '../../../support';
+
+import PageObjects from '../../../support/page_objects';
+
+var expect = require('expect.js');
+
+bdd.describe('source filters', function describeIndexTests() {
+ bdd.before(function () {
+
+ var fromTime = '2015-09-19 06:31:44.000';
+ var toTime = '2015-09-23 18:31:44.000';
+
+ // delete .kibana index and update configDoc
+ return esClient.deleteAndUpdateConfigDoc({'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*'})
+ .then(function loadkibanaIndexPattern() {
+ PageObjects.common.debug('load kibana index with default index pattern');
+ return elasticDump.elasticLoad('visualize_source-filters','.kibana');
+ })
+ // and load a set of makelogs data
+ .then(function loadIfEmptyMakelogs() {
+ return scenarioManager.loadIfEmpty('logstashFunctional');
+ })
+ .then(function () {
+ PageObjects.common.debug('discover');
+ return PageObjects.common.navigateToApp('discover');
+ })
+ .then(function () {
+ PageObjects.common.debug('setAbsoluteRange');
+ return PageObjects.header.setAbsoluteRange(fromTime, toTime);
+ })
+ .then(function () {
+ //After hiding the time picker, we need to wait for
+ //the refresh button to hide before clicking the share button
+ return PageObjects.common.sleep(1000);
+ });
+ });
+
+ bdd.it('should not get the field referer', function () {
+ return PageObjects.discover.getAllFieldNames()
+ .then(function (fieldNames) {
+ expect(fieldNames).to.not.contain('referer');
+ const relatedContentFields = fieldNames.filter((fieldName) => fieldName.indexOf('relatedContent') === 0);
+ expect(relatedContentFields).to.have.length(0);
+ });
+ });
+});
diff --git a/test/functional/apps/discover/index.js b/test/functional/apps/discover/index.js
index a4e16e769fd4e..a06ad49f8627a 100644
--- a/test/functional/apps/discover/index.js
+++ b/test/functional/apps/discover/index.js
@@ -23,4 +23,5 @@ bdd.describe('discover app', function () {
require('./_field_data');
require('./_shared_links');
require('./_collapse_expand');
+ require('./_source_filters');
});
diff --git a/test/functional/apps/management/_index_pattern_create_delete.js b/test/functional/apps/management/_index_pattern_create_delete.js
index e111ea706d2ce..db6c07079d4c3 100644
--- a/test/functional/apps/management/_index_pattern_create_delete.js
+++ b/test/functional/apps/management/_index_pattern_create_delete.js
@@ -55,10 +55,10 @@ bdd.describe('creating and deleting default index', function describeIndexTests(
'searchable',
'aggregatable',
'analyzed',
+ 'excluded',
'controls'
];
- // 6 name type format analyzed indexed controls
expect(headers.length).to.be(expectedHeaders.length);
var comparedHeaders = headers.map(function compareHead(header, i) {
diff --git a/test/support/page_objects/discover_page.js b/test/support/page_objects/discover_page.js
index 84c995b05e1a3..ae26e6001d483 100644
--- a/test/support/page_objects/discover_page.js
+++ b/test/support/page_objects/discover_page.js
@@ -229,6 +229,14 @@ export default class DiscoverPage {
.click();
}
+ getAllFieldNames() {
+ return this.findTimeout
+ .findAllByClassName('sidebar-item')
+ .then((items) => {
+ return Promise.all(items.map((item) => item.getVisibleText()));
+ });
+ }
+
getSidebarWidth() {
return this.findTimeout
.findByClassName('sidebar-list')