[Maps] Add mvt format for ES-doc sources

x-pack/package.json

@@ -267,6 +267,7 @@
     "font-awesome": "4.7.0",
     "formsy-react": "^1.1.5",
     "fp-ts": "^2.3.1",
+    "geojson-vt": "^3.2.1",
     "get-port": "^4.2.0",
     "getos": "^3.1.0",
     "git-url-parse": "11.1.2",
@@ -382,6 +383,7 @@
     "uuid": "3.3.2",
     "venn.js": "0.2.20",
     "vscode-languageserver": "^5.2.1",
+    "vt-pbf": "^3.1.1",
     "webpack": "^4.41.5",
     "wellknown": "^0.5.0",
     "xml2js": "^0.4.22",

x-pack/plugins/maps/common/constants.ts

@@ -32,6 +32,12 @@ export const MAP_PATH = 'map';
 export const GIS_API_PATH = `api/${APP_ID}`;
 export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`;
 export const FONTS_API_PATH = `${GIS_API_PATH}/fonts`;
+export const API_ROOT_PATH = `/${GIS_API_PATH}`;
+
+export const MVT_GETTILE_API_PATH = 'mvt/getTile';
+export const MVT_SOURCE_LAYER_NAME = 'geojsonLayer';
+export const KBN_TOO_MANY_FEATURES_PROPERTY = '__kbn_too_many_features__';
+export const KBN_TOO_MANY_FEATURES_IMAGE_ID = '__kbn_too_many_features_image_id__';
 
 const MAP_BASE_URL = `/${MAPS_APP_PATH}/${MAP_PATH}`;
 export function getNewMapPath() {
@@ -219,6 +225,7 @@ export enum SCALING_TYPES {
   LIMIT = 'LIMIT',
   CLUSTERS = 'CLUSTERS',
   TOP_HITS = 'TOP_HITS',
+  MVT = 'MVT',
 }
 
 export const RGBA_0000 = 'rgba(0,0,0,0)';

x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx

@@ -77,7 +77,7 @@ export class TiledVectorLayer extends VectorLayer {
 
     startLoading(SOURCE_DATA_REQUEST_ID, requestToken, searchFilters);
     try {
-      const templateWithMeta = await this._source.getUrlTemplateWithMeta();
+      const templateWithMeta = await this._source.getUrlTemplateWithMeta(dataFilters);
       stopLoading(SOURCE_DATA_REQUEST_ID, requestToken, templateWithMeta, {});
     } catch (error) {
       onLoadError(SOURCE_DATA_REQUEST_ID, requestToken, error.message);
@@ -160,6 +160,11 @@ export class TiledVectorLayer extends VectorLayer {
       return false;
     }
 
+    if (!mbTileSource.tiles) {
+      // Expected source is not compatible, so remove.
+      return true;
+    }
+
     const isSourceDifferent =
       mbTileSource.tiles[0] !== tiledSourceMeta.urlTemplate ||
       mbTileSource.minzoom !== tiledSourceMeta.minSourceZoom ||

x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js

@@ -15,9 +15,11 @@ import {
   SOURCE_BOUNDS_DATA_REQUEST_ID,
   FEATURE_VISIBLE_PROPERTY_NAME,
   EMPTY_FEATURE_COLLECTION,
+  KBN_TOO_MANY_FEATURES_PROPERTY,
   LAYER_TYPE,
   FIELD_ORIGIN,
   LAYER_STYLE_TYPE,
+  KBN_TOO_MANY_FEATURES_IMAGE_ID,
 } from '../../../../common/constants';
 import _ from 'lodash';
 import { JoinTooltipProperty } from '../../tooltips/join_tooltip_property';
@@ -777,6 +779,8 @@ export class VectorLayer extends AbstractLayer {
     const sourceId = this.getId();
     const fillLayerId = this._getMbPolygonLayerId();
     const lineLayerId = this._getMbLineLayerId();
+    const tooManyFeaturesLayerId = this._getMbTooManyFeaturesLayerId();
+
     const hasJoins = this.hasJoins();
     if (!mbMap.getLayer(fillLayerId)) {
       const mbLayer = {
@@ -802,6 +806,30 @@ export class VectorLayer extends AbstractLayer {
       }
       mbMap.addLayer(mbLayer);
     }
+    if (!mbMap.getLayer(tooManyFeaturesLayerId)) {
+      const mbLayer = {
+        id: tooManyFeaturesLayerId,
+        type: 'fill',
+        source: sourceId,
+        paint: {},
+      };
+      if (mvtSourceLayer) {
+        mbLayer['source-layer'] = mvtSourceLayer;
+      }
+      mbMap.addLayer(mbLayer);
+      mbMap.setFilter(tooManyFeaturesLayerId, [
+        '==',
+        ['get', KBN_TOO_MANY_FEATURES_PROPERTY],
+        true,
+      ]);
+      mbMap.setPaintProperty(
+        tooManyFeaturesLayerId,
+        'fill-pattern',
+        KBN_TOO_MANY_FEATURES_IMAGE_ID
+      );
+      mbMap.setPaintProperty(tooManyFeaturesLayerId, 'fill-opacity', this.getAlpha());
+    }
+
     this.getCurrentStyle().setMBPaintProperties({
       alpha: this.getAlpha(),
       mbMap,
@@ -822,6 +850,9 @@ export class VectorLayer extends AbstractLayer {
     if (lineFilterExpr !== mbMap.getFilter(lineLayerId)) {
       mbMap.setFilter(lineLayerId, lineFilterExpr);
     }
+
+    this.syncVisibilityWithMb(mbMap, tooManyFeaturesLayerId);
+    mbMap.setLayerZoomRange(tooManyFeaturesLayerId, this.getMinZoom(), this.getMaxZoom());
   }
 
   _syncStylePropertiesWithMb(mbMap) {
@@ -836,6 +867,18 @@ export class VectorLayer extends AbstractLayer {
         type: 'geojson',
         data: EMPTY_FEATURE_COLLECTION,
       });
+    } else if (mbSource.type !== 'geojson') {
+      this.getMbLayerIds().forEach((mbLayerId) => {
+        if (mbMap.getLayer(mbLayerId)) {
+          mbMap.removeLayer(mbLayerId);
+        }
+      });
+
+      mbMap.removeSource(this._getMbSourceId());
+      mbMap.addSource(this._getMbSourceId(), {
+        type: 'geojson',
+        data: EMPTY_FEATURE_COLLECTION,
+      });
     }
   }
 
@@ -865,13 +908,18 @@ export class VectorLayer extends AbstractLayer {
     return this.makeMbLayerId('fill');
   }
 
+  _getMbTooManyFeaturesLayerId() {
+    return this.makeMbLayerId('toomanyfeatures');
+  }
+
   getMbLayerIds() {
     return [
       this._getMbPointLayerId(),
       this._getMbTextLayerId(),
       this._getMbSymbolLayerId(),
       this._getMbLineLayerId(),
       this._getMbPolygonLayerId(),
+      this._getMbTooManyFeaturesLayerId(),
     ];
   }
 

x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx

@@ -14,13 +14,18 @@ import { ESSearchSource, sourceTitle } from './es_search_source';
 import { BlendedVectorLayer } from '../../layers/blended_vector_layer/blended_vector_layer';
 import { VectorLayer } from '../../layers/vector_layer/vector_layer';
 import { LAYER_WIZARD_CATEGORY, SCALING_TYPES } from '../../../../common/constants';
+import { TiledVectorLayer } from '../../layers/tiled_vector_layer/tiled_vector_layer';
 
 export function createDefaultLayerDescriptor(sourceConfig: unknown, mapColors: string[]) {
   const sourceDescriptor = ESSearchSource.createDescriptor(sourceConfig);
 
-  return sourceDescriptor.scalingType === SCALING_TYPES.CLUSTERS
-    ? BlendedVectorLayer.createDescriptor({ sourceDescriptor }, mapColors)
-    : VectorLayer.createDescriptor({ sourceDescriptor }, mapColors);
+  if (sourceDescriptor.scalingType === SCALING_TYPES.CLUSTERS) {
+    return BlendedVectorLayer.createDescriptor({ sourceDescriptor }, mapColors);
+  } else if (sourceDescriptor.scalingType === SCALING_TYPES.MVT) {
+    return TiledVectorLayer.createDescriptor({ sourceDescriptor }, mapColors);
+  } else {
+    return VectorLayer.createDescriptor({ sourceDescriptor }, mapColors);
+  }
 }
 
 export const esDocumentsLayerWizardConfig: LayerWizard = {

x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.d.ts

@@ -6,8 +6,9 @@
 
 import { AbstractESSource } from '../es_source';
 import { ESSearchSourceDescriptor } from '../../../../common/descriptor_types';
+import { ITiledSingleLayerVectorSource } from '../vector_source';
 
-export class ESSearchSource extends AbstractESSource {
+export class ESSearchSource extends AbstractESSource implements ITiledSingleLayerVectorSource {
   static createDescriptor(sourceConfig: unknown): ESSearchSourceDescriptor;
 
   constructor(sourceDescriptor: Partial<ESSearchSourceDescriptor>, inspectorAdapters: unknown);

x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.js

@@ -6,9 +6,10 @@
 
 import _ from 'lodash';
 import React from 'react';
+import rison from 'rison-node';
 
 import { AbstractESSource } from '../es_source';
-import { getSearchService } from '../../../kibana_services';
+import { getSearchService, getHttp } from '../../../kibana_services';
 import { hitsToGeoJson } from '../../../elasticsearch_geo_utils';
 import { UpdateSourceEditor } from './update_source_editor';
 import {
@@ -18,6 +19,11 @@ import {
   SORT_ORDER,
   SCALING_TYPES,
   VECTOR_SHAPE_TYPE,
+  MVT_SOURCE_LAYER_NAME,
+  GIS_API_PATH,
+  MVT_GETTILE_API_PATH,
+  MIN_ZOOM,
+  MAX_ZOOM,
 } from '../../../../common/constants';
 import { i18n } from '@kbn/i18n';
 import { getDataSourceLabel } from '../../../../common/i18n_getters';
@@ -448,9 +454,13 @@ export class ESSearchSource extends AbstractESSource {
   }
 
   isFilterByMapBounds() {
-    return this._descriptor.scalingType === SCALING_TYPES.CLUSTER
-      ? true
-      : this._descriptor.filterByMapBounds;
+    if (this._descriptor.scalingType === SCALING_TYPES.CLUSTER) {
+      return true;
+    } else if (this._descriptor.scalingType === SCALING_TYPES.MVT) {
+      return false;
+    } else {
+      return this._descriptor.filterByMapBounds;
+    }
   }
 
   async getLeftJoinFields() {
@@ -553,11 +563,66 @@ export class ESSearchSource extends AbstractESSource {
   }
 
   getJoinsDisabledReason() {
-    return this._descriptor.scalingType === SCALING_TYPES.CLUSTERS
-      ? i18n.translate('xpack.maps.source.esSearch.joinsDisabledReason', {
-          defaultMessage: 'Joins are not supported when scaling by clusters',
-        })
-      : null;
+    let reason;
+    if (this._descriptor.scalingType === SCALING_TYPES.CLUSTERS) {
+      reason = i18n.translate('xpack.maps.source.esSearch.joinsDisabledReason', {
+        defaultMessage: 'Joins are not supported when scaling by clusters',
+      });
+    } else if (this._descriptor.scalingType === SCALING_TYPES.MVT) {
+      reason = i18n.translate('xpack.maps.source.esSearch.joinsDisabledReasonMvt', {
+        defaultMessage: 'Joins are not supported when scaling by mvt vector tiles',
+      });
+    } else {
+      reason = null;
+    }
+    return reason;
+  }
+
+  // MVT methods. Do we really need different source-type?????
+  getLayerName() {
+    return 'water';
+  }
+
+  async getUrlTemplateWithMeta(searchFilters) {
+    const indexPattern = await this.getIndexPattern();
+    const indexSettings = await loadIndexSettings(indexPattern.title);
+
+    //assuming only geo_shape fields for now
+    const initialSearchContext = {
+      // docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames),
+    };
+    // const geoField = await this._getGeoField();
+    // const docValueFields = await this._excludeDateFields(searchFilters.fieldNames);
+    // const withoutGeoField = docValueFields.filter((field) => field !== geoField.name);
+    // initialSearchContext.docvalue_fields.push(...withoutGeoField);
+
+    const searchSource = await this.makeSearchSource(
+      searchFilters,
+      indexSettings.maxResultWindow,
+      initialSearchContext
+    );
+    searchSource.setField('fields', searchFilters.fieldNames);
+
+    const ipTitle = indexPattern.title;
+    const geometryFieldName = this._descriptor.geoField;
+    const fields = ['_id']; //todo needs to include correct fields
+    const fieldsParam = fields.join(',');
+
+    const dsl = await searchSource.getSearchRequestBody();
+
+    const risonDsl = rison.encode(dsl);
+
+    const mvtUrlServicePath = getHttp().basePath.prepend(
+      `/${GIS_API_PATH}/${MVT_GETTILE_API_PATH}`
+    );
+
+    const urlTemplate = `${mvtUrlServicePath}?x={x}&y={y}&z={z}&geometryFieldName=${geometryFieldName}&indexPattern=${ipTitle}&fields=${fieldsParam}&requestBody=${risonDsl}`;
+    return {
+      layerName: MVT_SOURCE_LAYER_NAME,
+      minSourceZoom: MIN_ZOOM,
+      maxSourceZoom: MAX_ZOOM,
+      urlTemplate: urlTemplate,
+    };
   }
 }