added source filtering

src/plugins/kibana/public/settings/sections/indices/_indexed_fields.js

@@ -3,7 +3,7 @@ define(function (require) {
   require('ui/paginated_table');
 
   require('ui/modules').get('apps/settings')
-  .directive('indexedFields', function ($filter) {
+  .directive('indexedFields', function ($filter, config) {
     var yesTemplate = '<i class="fa fa-check" aria-label="yes"></i>';
     var noTemplate = '';
     var nameHtml = require('plugins/kibana/settings/sections/indices/_field_name.html');
@@ -24,6 +24,7 @@ define(function (require) {
           { title: 'format' },
           { title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
           { title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
+          { title: 'exclude', info: 'Fields that are excluded from _source when it is fetched' },
           { title: 'controls', sortable: false }
         ];
 
@@ -60,6 +61,10 @@ define(function (require) {
                 markup: field.indexed ? yesTemplate : noTemplate,
                 value: field.indexed
               },
+              {
+                markup: field.exclude ? yesTemplate : noTemplate,
+                value: field.exclude
+              },
               {
                 markup: controlsHtml,
                 scope: childScope

src/plugins/kibana/server/lib/init_default_field_props.js

@@ -17,6 +17,7 @@ module.exports = function initDefaultFieldProps(fields) {
         analyzed: true,
         doc_values: false,
         scripted: false,
+        exclude: false,
         count: 0
       });
 
@@ -27,6 +28,7 @@ module.exports = function initDefaultFieldProps(fields) {
         analyzed: false,
         doc_values: true,
         scripted: false,
+        exclude: false,
         count: 0
       });
     }
@@ -36,6 +38,7 @@ module.exports = function initDefaultFieldProps(fields) {
         analyzed: false,
         doc_values: true,
         scripted: false,
+        exclude: false,
         count: 0
       });
     }

src/testUtils/stub_index_pattern.js

@@ -18,6 +18,7 @@ define(function (require) {
       this.timeFieldName = timeField;
       this.getNonScriptedFields = sinon.spy();
       this.getScriptedFields = sinon.spy();
+      this.getSourceFiltering = sinon.spy();
       this.metaFields = ['_id', '_type', '_source'];
       this.fieldFormatMap = {};
       this.routes = IndexPattern.prototype.routes;

src/ui/public/doc_table/__tests__/doc_table.js

@@ -71,6 +71,10 @@ describe('docTable', function () {
     expect($elem.text()).to.not.be.empty();
   });
 
+  it('should set the source filtering defintion', function () {
+    expect($scope.indexPattern.getSourceFiltering.called).to.be(true);
+  });
+
   it('should set the indexPattern to that of the searchSource', function () {
     expect($scope.indexPattern).to.be(searchSource.get('index'));
   });

src/ui/public/doc_table/doc_table.js

@@ -81,6 +81,11 @@ define(function (require) {
           $scope.searchSource.size(config.get('discover:sampleSize'));
           $scope.searchSource.sort(getSort($scope.sorting, $scope.indexPattern));
 
+          var sourceFiltering = $scope.indexPattern.getSourceFiltering($scope.columns);
+          if (sourceFiltering) {
+            $scope.searchSource.source(sourceFiltering);
+          }
+
           // Set the watcher after initialization
           $scope.$watchCollection('sorting', function (newSort, oldSort) {
             // Don't react if sort values didn't really change

src/ui/public/field_editor/field_editor.html

@@ -83,6 +83,38 @@ <h4 class="hintbox-heading">
     </div>
   </div>
 
+  <div class="form-group">
+    <span class="pull-right text-info hintbox-label" ng-click="editor.showExcludeHelp = !editor.showExcludeHelp">
+      <i class="fa fa-info"></i> Help
+    </span>
+
+    <label><input ng-disabled="isMetaField" type="checkbox" ng-model="editor.field.exclude"> Exclude from _source</label>
+
+    <div class="hintbox" ng-if="isMetaField">
+      <h4 class="hintbox-heading">
+        <i class="fa fa-warning text-warning"></i> Field Exclusion
+      </h4>
+
+      <p>
+      You cannot exclude metadata fields.
+      </p>
+    </div>
+
+    <div class="hintbox" ng-if="editor.showExcludeHelp">
+      <h4 class="hintbox-heading">
+        <i class="fa fa-info text-info"></i> Field Exclusion
+      </h4>
+
+      <p>
+      If checked, this field will be filtered from the <b>_source</b> of each document using
+      <a target="_window" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html">source filtering<i aria-hidden="true" class="fa-link fa"></i></a>. This only impacts views that fetch the _source for documents, like Discover or the doc table.
+      </p>
+      <p>
+      If this field is explicitly selected in your saved search, then it will not be excluded.
+      </p>
+    </div>
+  </div>
+
   <div ng-if="editor.field.scripted">
     <div class="form-group">
       <label>Script</label>

src/ui/public/field_editor/field_editor.js

@@ -21,16 +21,19 @@ define(function (require) {
         getField: '&field'
       },
       controllerAs: 'editor',
-      controller: function ($scope, Notifier, kbnUrl) {
+      controller: function ($scope, Notifier, kbnUrl, config) {
         var self = this;
         var notify = new Notifier({ location: 'Field Editor' });
 
+        const metaFields = config.get('metaFields');
+
         self.scriptingInfo = scriptingInfo;
         self.scriptingWarning = scriptingWarning;
 
         self.indexPattern = $scope.getIndexPattern();
         self.field = shadowCopy($scope.getField());
         self.formatParams = self.field.format.params();
+        $scope.isMetaField = _.contains(metaFields, self.field.name);
 
         // only init on first create
         self.creating = !self.indexPattern.fields.byName[self.field.name];

src/ui/public/index_patterns/_field.js

@@ -1,5 +1,5 @@
 define(function (require) {
-  return function FieldObjectProvider(Private, shortDotsFilter, $rootScope, Notifier) {
+  return function FieldObjectProvider(Private, shortDotsFilter, $rootScope, Notifier, config) {
     var notify = new Notifier({ location: 'IndexPattern Field' });
     var FieldFormat = Private(require('ui/index_patterns/_field_format/FieldFormat'));
     var fieldTypes = Private(require('ui/index_patterns/_field_types'));
@@ -41,10 +41,12 @@ define(function (require) {
       var sortable = spec.name === '_score' || ((indexed || scripted) && type.sortable);
       var bucketable = indexed || scripted;
       var filterable = spec.name === '_id' || scripted || (indexed && type.filterable);
+      var isMetaField = config.get('metaFields').includes(spec.name);
 
       obj.fact('name');
       obj.fact('type');
       obj.writ('count', spec.count || 0);
+      obj.fact('exclude', Boolean(!isMetaField && spec.exclude));
 
       // scripted objs
       obj.fact('scripted', scripted);

src/ui/public/index_patterns/_index_pattern.js

@@ -122,6 +122,17 @@ define(function (require) {
         }
       };
 
+      // Get the source filtering configuration for that index.
+      // Fields which name appears in the given columns array will not be excluded.
+      self.getSourceFiltering = function (columns) {
+        return {
+          exclude: _(self.getNonScriptedFields())
+            .filter((field) => field.exclude && !_.contains(columns, field.name))
+            .map((field) => field.name)
+            .value()
+        };
+      };
+
       self.addScriptedField = function (name, script, type, lang) {
         type = type || 'string';
 
@@ -282,8 +293,19 @@ define(function (require) {
       };
 
       self._fetchFields = function () {
+        var existingFieldsByName = self.fields.byName;
+
         return mapper.getFieldsForIndexPattern(self, true)
         .then(function (fields) {
+
+          // copy over kibana-added properties from existing fields
+          fields.forEach(function (field) {
+            var existingField = existingFieldsByName[field.name];
+            if (existingField) {
+              field.exclude = existingField.exclude;
+            }
+          });
+
           // append existing scripted fields
           fields = fields.concat(self.getScriptedFields());
 

test/functional/apps/settings/_index_pattern_create_delete.js

@@ -53,6 +53,7 @@ define(function (require) {
               'format',
               'analyzed',
               'indexed',
+              'exclude',
               'controls'
             ];