diff --git a/docs/development/core/server/kibana-plugin-core-server.kibanaresponsefactory.md b/docs/development/core/server/kibana-plugin-core-server.kibanaresponsefactory.md index 91cb6c370d759..d5ba98c06ef02 100644 --- a/docs/development/core/server/kibana-plugin-core-server.kibanaresponsefactory.md +++ b/docs/development/core/server/kibana-plugin-core-server.kibanaresponsefactory.md @@ -34,7 +34,7 @@ kibanaResponseFactory: { message: string | Error; attributes?: ResponseErrorAttributes | undefined; }>; - customError: (options: CustomHttpResponseOptions) => KibanaResponse) => KibanaResponse; diff --git a/src/core/server/http/router/response.ts b/src/core/server/http/router/response.ts index 331a1ab38f069..eebd0d0dd6eef 100644 --- a/src/core/server/http/router/response.ts +++ b/src/core/server/http/router/response.ts @@ -186,7 +186,7 @@ const errorResponseFactory = { * Creates an error response with defined status code and payload. * @param options - {@link CustomHttpResponseOptions} configures HTTP response headers, error message and other error details to pass to the client */ - customError: (options: CustomHttpResponseOptions) => { + customError: (options: CustomHttpResponseOptions) => { if (!options || !options.statusCode) { throw new Error( `options.statusCode is expected to be set. given options: ${options && options.statusCode}` diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 42606a4ad85da..0a48e4dc6380a 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1464,7 +1464,7 @@ export const kibanaResponseFactory: { message: string | Error; attributes?: ResponseErrorAttributes | undefined; }>; - customError: (options: CustomHttpResponseOptions) => KibanaResponse) => KibanaResponse; diff --git a/x-pack/plugins/maps/public/actions/data_request_actions.ts b/x-pack/plugins/maps/public/actions/data_request_actions.ts index 730135424a4dd..5788c9360fdf2 100644 --- a/x-pack/plugins/maps/public/actions/data_request_actions.ts +++ b/x-pack/plugins/maps/public/actions/data_request_actions.ts @@ -220,7 +220,7 @@ export function syncDataForLayerId(layerId: string | null, isForceRefresh: boole }; } -function setLayerDataLoadErrorStatus(layerId: string, errorMessage: string | null) { +export function setLayerDataLoadErrorStatus(layerId: string, errorMessage: string | null) { return { type: SET_LAYER_ERROR_STATUS, isInErrorState: errorMessage !== null, diff --git a/x-pack/plugins/maps/public/actions/index.ts b/x-pack/plugins/maps/public/actions/index.ts index f4d6997333c6c..2568a787b3941 100644 --- a/x-pack/plugins/maps/public/actions/index.ts +++ b/x-pack/plugins/maps/public/actions/index.ts @@ -15,6 +15,7 @@ export { cancelAllInFlightRequests, fitToLayerExtent, fitToDataBounds, + setLayerDataLoadErrorStatus, } from './data_request_actions'; export { closeOnClickTooltip, diff --git a/x-pack/plugins/maps/public/classes/util/geo_tile_utils.test.ts b/x-pack/plugins/maps/public/classes/util/geo_tile_utils.test.ts index f391d3bfd3307..656ded611a115 100644 --- a/x-pack/plugins/maps/public/classes/util/geo_tile_utils.test.ts +++ b/x-pack/plugins/maps/public/classes/util/geo_tile_utils.test.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { parseTileKey, getTileBoundingBox, expandToTileBoundaries } from './geo_tile_utils'; +import { + getTileKey, + parseTileKey, + getTileBoundingBox, + expandToTileBoundaries, +} from './geo_tile_utils'; it('Should parse tile key', () => { expect(parseTileKey('15/23423/1867')).toEqual({ @@ -16,6 +21,10 @@ it('Should parse tile key', () => { }); }); +it('Should get tile key', () => { + expect(getTileKey(45, 120, 10)).toEqual('10/853/368'); +}); + it('Should convert tile key to geojson Polygon', () => { const geometry = getTileBoundingBox('15/23423/1867'); expect(geometry).toEqual({ diff --git a/x-pack/plugins/maps/public/classes/util/geo_tile_utils.ts b/x-pack/plugins/maps/public/classes/util/geo_tile_utils.ts index 36c7d9d6c4a11..f3fe55b5e47c6 100644 --- a/x-pack/plugins/maps/public/classes/util/geo_tile_utils.ts +++ b/x-pack/plugins/maps/public/classes/util/geo_tile_utils.ts @@ -60,6 +60,14 @@ export function parseTileKey(tileKey: string): { return { x, y, zoom, tileCount }; } +export function getTileKey(lat: number, lon: number, zoom: number): string { + const tileCount = getTileCount(zoom); + + const x = longitudeToTile(lon, tileCount); + const y = latitudeToTile(lat, tileCount); + return `${zoom}/${x}/${y}`; +} + function sinh(x: number): number { return (Math.exp(x) - Math.exp(-x)) / 2; } diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/index.ts b/x-pack/plugins/maps/public/connected_components/mb_map/index.ts index df03f755d6d2b..3c9244a74914d 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/index.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/index.ts @@ -16,6 +16,7 @@ import { mapExtentChanged, mapReady, setAreTilesLoaded, + setLayerDataLoadErrorStatus, setMapInitError, setMouseCoordinates, updateMetaFromTiles, @@ -86,6 +87,12 @@ function mapDispatchToProps(dispatch: ThunkDispatch void; featureModeActive: boolean; filterModeActive: boolean; + setTileLoadError(layerId: string, errorMessage: string): void; + clearTileLoadError(layerId: string): void; } interface State { @@ -205,8 +207,15 @@ export class MbMap extends Component { this._tileStatusTracker = new TileStatusTracker({ mbMap, getCurrentLayerList: () => this.props.layerList, - updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => { + updateTileStatus: (layer: ILayer, areTilesLoaded: boolean, errorMessage?: string) => { this.props.setAreTilesLoaded(layer.getId(), areTilesLoaded); + + if (errorMessage) { + this.props.setTileLoadError(layer.getId(), errorMessage); + } else { + this.props.clearTileLoadError(layer.getId()); + } + this._queryForMeta(layer); }, }); diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts index 76052f4590aba..ffc6459262c8b 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.test.ts @@ -61,6 +61,11 @@ function createMockMbDataEvent(mbSourceId: string, tileKey: string): unknown { dataType: 'source', tile: { tileID: { + canonical: { + x: 80, + y: 10, + z: 5, + }, key: tileKey, }, }, @@ -133,7 +138,7 @@ describe('TileStatusTracker', () => { }, }); - expect(mockMbMap.listeners.length).toBe(3); + expect(mockMbMap.listeners.length).toBe(4); tileStatusTracker.destroy(); expect(mockMbMap.listeners.length).toBe(0); }); diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts index 72d692a147ff2..94a4344bac009 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tile_status_tracker.ts @@ -7,14 +7,21 @@ import type { Map as MapboxMap, MapSourceDataEvent } from '@kbn/mapbox-gl'; import _ from 'lodash'; +import { i18n } from '@kbn/i18n'; import { ILayer } from '../../classes/layers/layer'; import { SPATIAL_FILTERS_LAYER_ID } from '../../../common/constants'; +import { getTileKey } from '../../classes/util/geo_tile_utils'; interface MbTile { // references internal object from mapbox aborted?: boolean; } +type TileError = Error & { + status: number; + tileZXYKey: string; // format zoom/x/y +}; + interface Tile { mbKey: string; mbSourceId: string; @@ -23,9 +30,16 @@ interface Tile { export class TileStatusTracker { private _tileCache: Tile[]; + private _tileErrorCache: Record; + private _prevCenterTileKey?: string; private readonly _mbMap: MapboxMap; - private readonly _updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => void; + private readonly _updateTileStatus: ( + layer: ILayer, + areTilesLoaded: boolean, + errorMessage?: string + ) => void; private readonly _getCurrentLayerList: () => ILayer[]; + private readonly _onSourceDataLoading = (e: MapSourceDataEvent) => { if ( e.sourceId && @@ -51,16 +65,29 @@ export class TileStatusTracker { } }; - private readonly _onError = (e: MapSourceDataEvent) => { + private readonly _onError = (e: MapSourceDataEvent & { error: Error & { status: number } }) => { if ( e.sourceId && e.sourceId !== SPATIAL_FILTERS_LAYER_ID && e.tile && (e.source.type === 'vector' || e.source.type === 'raster') ) { + const targetLayer = this._getCurrentLayerList().find((layer) => { + return layer.ownsMbSourceId(e.sourceId); + }); + const layerId = targetLayer ? targetLayer.getId() : undefined; + if (layerId) { + const layerErrors = this._tileErrorCache[layerId] ? this._tileErrorCache[layerId] : []; + layerErrors.push({ + ...e.error, + tileZXYKey: `${e.tile.tileID.canonical.z}/${e.tile.tileID.canonical.x}/${e.tile.tileID.canonical.y}`, + } as TileError); + this._tileErrorCache[layerId] = layerErrors; + } this._removeTileFromCache(e.sourceId, e.tile.tileID.key as unknown as string); } }; + private readonly _onSourceData = (e: MapSourceDataEvent) => { if ( e.sourceId && @@ -73,16 +100,35 @@ export class TileStatusTracker { } }; + /* + * Clear errors when center tile changes. + * Tracking center tile provides the cleanest way to know when a new data fetching cycle is beginning + */ + private readonly _onMove = () => { + const center = this._mbMap.getCenter(); + // Maplibre rounds zoom when 'source.roundZoom' is true and floors zoom when 'source.roundZoom' is false + // 'source.roundZoom' is true for raster and video layers + // 'source.roundZoom' is false for vector layers + // Always floor zoom to keep logic as simple as possible and not have to track center tile by source. + // We are mainly concerned with showing errors from Elasticsearch vector tile requests (which are vector sources) + const centerTileKey = getTileKey(center.lat, center.lng, Math.floor(this._mbMap.getZoom())); + if (this._prevCenterTileKey !== centerTileKey) { + this._prevCenterTileKey = centerTileKey; + this._tileErrorCache = {}; + } + }; + constructor({ mbMap, updateTileStatus, getCurrentLayerList, }: { mbMap: MapboxMap; - updateTileStatus: (layer: ILayer, areTilesLoaded: boolean) => void; + updateTileStatus: (layer: ILayer, areTilesLoaded: boolean, errorMessage?: string) => void; getCurrentLayerList: () => ILayer[]; }) { this._tileCache = []; + this._tileErrorCache = {}; this._updateTileStatus = updateTileStatus; this._getCurrentLayerList = getCurrentLayerList; @@ -90,6 +136,7 @@ export class TileStatusTracker { this._mbMap.on('sourcedataloading', this._onSourceDataLoading); this._mbMap.on('error', this._onError); this._mbMap.on('sourcedata', this._onSourceData); + this._mbMap.on('move', this._onMove); } _updateTileStatusForAllLayers = _.debounce(() => { @@ -107,7 +154,31 @@ export class TileStatusTracker { break; } } - this._updateTileStatus(layer, !atLeastOnePendingTile); + const tileErrorMessages = this._tileErrorCache[layer.getId()] + ? this._tileErrorCache[layer.getId()].map((tileError) => { + return i18n.translate('xpack.maps.tileStatusTracker.tileErrorMsg', { + defaultMessage: `tile '{tileZXYKey}' failed to load: '{status} {message}'`, + values: { + tileZXYKey: tileError.tileZXYKey, + status: tileError.status, + message: tileError.message, + }, + }); + }) + : []; + this._updateTileStatus( + layer, + !atLeastOnePendingTile, + tileErrorMessages.length + ? i18n.translate('xpack.maps.tileStatusTracker.layerErrorMsg', { + defaultMessage: `Unable to load {count} tiles: {tileErrors}`, + values: { + count: tileErrorMessages.length, + tileErrors: tileErrorMessages.join(', '), + }, + }) + : undefined + ); } }, 100); @@ -126,6 +197,7 @@ export class TileStatusTracker { this._mbMap.off('error', this._onError); this._mbMap.off('sourcedata', this._onSourceData); this._mbMap.off('sourcedataloading', this._onSourceDataLoading); + this._mbMap.off('move', this._onMove); this._tileCache.length = 0; } } diff --git a/x-pack/plugins/maps/server/mvt/get_grid_tile.ts b/x-pack/plugins/maps/server/mvt/get_grid_tile.ts index 52232ff6533cc..e88a0f0314931 100644 --- a/x-pack/plugins/maps/server/mvt/get_grid_tile.ts +++ b/x-pack/plugins/maps/server/mvt/get_grid_tile.ts @@ -41,7 +41,7 @@ export async function getEsGridTile({ renderAs: RENDER_AS; gridPrecision: number; abortController: AbortController; -}): Promise<{ stream: Stream | null; headers?: IncomingHttpHeaders }> { +}): Promise<{ stream: Stream | null; headers: IncomingHttpHeaders; statusCode: number }> { try { const path = `/${encodeURIComponent(index)}/_mvt/${geometryFieldName}/${z}/${x}/${y}`; const body = { @@ -81,13 +81,15 @@ export async function getEsGridTile({ } ); - return { stream: tile.body as Stream, headers: tile.headers }; + return { stream: tile.body as Stream, headers: tile.headers, statusCode: tile.statusCode }; } catch (e) { - if (!isAbortError(e)) { - // These are often circuit breaking exceptions - // Should return a tile with some error message - logger.warn(`Cannot generate ES-grid-tile for ${z}/${x}/${y}: ${e.message}`); + if (isAbortError(e)) { + return { stream: null, headers: {}, statusCode: 200 }; } - return { stream: null }; + + // These are often circuit breaking exceptions + // Should return a tile with some error message + logger.warn(`Cannot generate ES-grid-tile for ${z}/${x}/${y}: ${e.message}`); + return { stream: null, headers: {}, statusCode: 500 }; } } diff --git a/x-pack/plugins/maps/server/mvt/get_tile.ts b/x-pack/plugins/maps/server/mvt/get_tile.ts index 7070b3a04bf2b..b4ae9e6b698d6 100644 --- a/x-pack/plugins/maps/server/mvt/get_tile.ts +++ b/x-pack/plugins/maps/server/mvt/get_tile.ts @@ -37,7 +37,7 @@ export async function getEsTile({ logger: Logger; requestBody: any; abortController: AbortController; -}): Promise<{ stream: Stream | null; headers?: IncomingHttpHeaders }> { +}): Promise<{ stream: Stream | null; headers: IncomingHttpHeaders; statusCode: number }> { try { const path = `/${encodeURIComponent(index)}/_mvt/${geometryFieldName}/${z}/${x}/${y}`; @@ -81,13 +81,15 @@ export async function getEsTile({ } ); - return { stream: tile.body as Stream, headers: tile.headers }; + return { stream: tile.body as Stream, headers: tile.headers, statusCode: tile.statusCode }; } catch (e) { - if (!isAbortError(e)) { - // These are often circuit breaking exceptions - // Should return a tile with some error message - logger.warn(`Cannot generate ES-grid-tile for ${z}/${x}/${y}: ${e.message}`); + if (isAbortError(e)) { + return { stream: null, headers: {}, statusCode: 200 }; } - return { stream: null }; + + // These are often circuit breaking exceptions + // Should return a tile with some error message + logger.warn(`Cannot generate ES-grid-tile for ${z}/${x}/${y}: ${e.message}`); + return { stream: null, headers: {}, statusCode: 500 }; } } diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.test.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.test.ts new file mode 100644 index 0000000000000..9d9ef24ef4795 --- /dev/null +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import sinon from 'sinon'; +import { KibanaResponseFactory } from '@kbn/core/server'; +import { sendResponse } from './mvt_routes'; + +const mockStream = Readable.from(['{}']); + +test('should send error response when status code is above 400', () => { + const responseMock = { + customError: sinon.spy(), + ok: sinon.spy(), + }; + sendResponse(responseMock as unknown as KibanaResponseFactory, mockStream, {}, 400); + expect(responseMock.ok.notCalled); + expect(responseMock.customError.calledOnce); + const firstCallArgs = responseMock.customError.getCall(0).args[0]; + expect(firstCallArgs.statusCode).toBe(400); +}); + +test('should forward content-length and content-encoding elasticsearch headers', () => { + const responseMock = { + customError: sinon.spy(), + ok: sinon.spy(), + }; + sendResponse( + responseMock as unknown as KibanaResponseFactory, + mockStream, + { 'content-encoding': 'gzip', 'content-length': '19326' }, + 200 + ); + expect(responseMock.ok.calledOnce); + expect(responseMock.customError.notCalled); + const firstCallArgs = responseMock.ok.getCall(0).args[0]; + const headers = { ...firstCallArgs.headers }; + + // remove lastModified from comparision check since its a timestamp that changes every run + expect(headers).toHaveProperty('Last-Modified'); + delete headers['Last-Modified']; + expect(headers).toEqual({ + 'Cache-Control': 'public, max-age=3600', + 'Content-Type': 'application/x-protobuf', + 'content-disposition': 'inline', + 'content-encoding': 'gzip', + 'content-length': '19326', + }); +}); + +test('should not set content-encoding when elasticsearch does not provide value', () => { + const responseMock = { + customError: sinon.spy(), + ok: sinon.spy(), + }; + sendResponse( + responseMock as unknown as KibanaResponseFactory, + mockStream, + { 'content-length': '19326' }, + 200 + ); + expect(responseMock.ok.calledOnce); + expect(responseMock.customError.notCalled); + const firstCallArgs = responseMock.ok.getCall(0).args[0]; + const headers = { ...firstCallArgs.headers }; + + // remove lastModified from comparision check since its a timestamp that changes every run + expect(headers).toHaveProperty('Last-Modified'); + delete headers['Last-Modified']; + expect(headers).toEqual({ + 'Cache-Control': 'public, max-age=3600', + 'Content-Type': 'application/x-protobuf', + 'content-disposition': 'inline', + 'content-length': '19326', + }); +}); diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.ts index c5bf0dfbc6df1..b7c6a59ba54d4 100644 --- a/x-pack/plugins/maps/server/mvt/mvt_routes.ts +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.ts @@ -58,7 +58,7 @@ export function initMVTRoutes({ const abortController = makeAbortController(request); - const { stream, headers } = await getEsTile({ + const { stream, headers, statusCode } = await getEsTile({ url: `${API_ROOT_PATH}/${MVT_GETTILE_API_PATH}/{z}/{x}/{y}.pbf`, core, logger, @@ -72,7 +72,7 @@ export function initMVTRoutes({ abortController, }); - return sendResponse(response, stream, headers); + return sendResponse(response, stream, headers, statusCode); } ); @@ -104,7 +104,7 @@ export function initMVTRoutes({ const abortController = makeAbortController(request); - const { stream, headers } = await getEsGridTile({ + const { stream, headers, statusCode } = await getEsGridTile({ url: `${API_ROOT_PATH}/${MVT_GETGRIDTILE_API_PATH}/{z}/{x}/{y}.pbf`, core, logger, @@ -120,23 +120,31 @@ export function initMVTRoutes({ abortController, }); - return sendResponse(response, stream, headers); + return sendResponse(response, stream, headers, statusCode); } ); } -function sendResponse( +export function sendResponse( response: KibanaResponseFactory, - gzipTileStream: Stream | null, - headers?: IncomingHttpHeaders + tileStream: Stream | null, + headers: IncomingHttpHeaders, + statusCode: number ) { + if (statusCode >= 400) { + return response.customError({ + statusCode, + body: tileStream ? tileStream : statusCode.toString(), + }); + } + const cacheControl = `public, max-age=${CACHE_TIMEOUT_SECONDS}`; const lastModified = `${new Date().toUTCString()}`; - if (gzipTileStream && headers) { + if (tileStream) { // use the content-encoding and content-length headers from elasticsearch if they exist const { 'content-length': contentLength, 'content-encoding': contentEncoding } = headers; return response.ok({ - body: gzipTileStream, + body: tileStream, headers: { 'content-disposition': 'inline', ...(contentLength && { 'content-length': contentLength }), diff --git a/x-pack/test/api_integration/apis/maps/get_grid_tile.js b/x-pack/test/api_integration/apis/maps/get_grid_tile.js index ab8c86215a3a5..38b34257d06ae 100644 --- a/x-pack/test/api_integration/apis/maps/get_grid_tile.js +++ b/x-pack/test/api_integration/apis/maps/get_grid_tile.js @@ -192,5 +192,13 @@ export default function ({ getService }) { ], ]); }); + + it('should return error when index does not exist', async () => { + await supertest + .get(URL.replace('index=logstash-*', 'index=notRealIndex') + '&renderAs=point') + .set('kbn-xsrf', 'kibana') + .responseType('blob') + .expect(404); + }); }); } diff --git a/x-pack/test/api_integration/apis/maps/get_tile.js b/x-pack/test/api_integration/apis/maps/get_tile.js index dd85fd094a804..9f85b6098f2be 100644 --- a/x-pack/test/api_integration/apis/maps/get_tile.js +++ b/x-pack/test/api_integration/apis/maps/get_tile.js @@ -84,5 +84,18 @@ export default function ({ getService }) { ], ]); }); + + it('should return error when index does not exist', async () => { + await supertest + .get( + `/api/maps/mvt/getTile/2/1/1.pbf\ +?geometryFieldName=geo.coordinates\ +&index=notRealIndex\ +&requestBody=(_source:!f,docvalue_fields:!(bytes,geo.coordinates,machine.os.raw,(field:'@timestamp',format:epoch_millis)),query:(bool:(filter:!((match_all:()),(range:(%27@timestamp%27:(format:strict_date_optional_time,gte:%272015-09-20T00:00:00.000Z%27,lte:%272015-09-20T01:00:00.000Z%27)))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(),size:10000,stored_fields:!(bytes,geo.coordinates,machine.os.raw,'@timestamp'))` + ) + .set('kbn-xsrf', 'kibana') + .responseType('blob') + .expect(404); + }); }); }