diff --git a/x-pack/legacy/plugins/apm/typings/elasticsearch.ts b/x-pack/legacy/plugins/apm/typings/elasticsearch.ts index 86ea7d3bcb4bd..7d63d1ede2022 100644 --- a/x-pack/legacy/plugins/apm/typings/elasticsearch.ts +++ b/x-pack/legacy/plugins/apm/typings/elasticsearch.ts @@ -28,7 +28,9 @@ declare module 'elasticsearch' { | 'extended_stats' | 'filter' | 'filters' - | 'cardinality'; + | 'cardinality' + | 'sampler' + | 'value_count'; type AggOptions = AggregationOptionMap & { [key: string]: any; @@ -71,6 +73,12 @@ declare module 'elasticsearch' { >; }; + type SamplerAggregation = SubAggregation< + SubAggregationMap + > & { + doc_count: number; + }; + interface AggregatedValue { value: number | null; } @@ -82,7 +90,9 @@ declare module 'elasticsearch' { max: AggregatedValue; min: AggregatedValue; sum: AggregatedValue; - terms: BucketAggregation; + value_count: AggregatedValue; + // Elasticsearch might return terms with numbers, but this is a more limited type + terms: BucketAggregation; date_histogram: BucketAggregation< AggregationOption[AggregationName], number @@ -128,6 +138,7 @@ declare module 'elasticsearch' { cardinality: { value: number; }; + sampler: SamplerAggregation; }[AggregationType & keyof AggregationOption[AggregationName]]; } >; diff --git a/x-pack/legacy/plugins/lens/common/api.ts b/x-pack/legacy/plugins/lens/common/api.ts new file mode 100644 index 0000000000000..f7e1c439074fc --- /dev/null +++ b/x-pack/legacy/plugins/lens/common/api.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface BucketedAggregation { + buckets: Array<{ + key: KeyType; + count: number; + }>; +} + +export interface NumberStatsResult { + count: number; + histogram: BucketedAggregation; + topValues: BucketedAggregation; +} + +export interface TopValuesResult { + count: number; + topValues: BucketedAggregation; +} + +export interface FieldStatsResponse { + // Total count of documents + totalDocuments?: number; + // If sampled, the exact number of matching documents + sampledDocuments?: number; + // If sampled, the exact number of values sampled. Can be higher than documents + // because Elasticsearch supports arrays for all fields + sampledValues?: number; + // Histogram and values are based on distinct values, not based on documents + histogram?: BucketedAggregation; + topValues?: BucketedAggregation; +} diff --git a/x-pack/legacy/plugins/lens/common/index.ts b/x-pack/legacy/plugins/lens/common/index.ts index 358d0d5b7e076..eead93dd33480 100644 --- a/x-pack/legacy/plugins/lens/common/index.ts +++ b/x-pack/legacy/plugins/lens/common/index.ts @@ -4,4 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './api'; export * from './constants'; diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss new file mode 100644 index 0000000000000..be957dbb403bf --- /dev/null +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/_field_itm.scss @@ -0,0 +1,21 @@ +.lnsFieldItem__topValue { + margin-bottom: $euiSizeS; + + &:last-of-type { + margin-bottom: 0; + } +} + +.lnsFieldItem__topValueProgress { + background-color: $euiColorLightestShade; + + &::-webkit-progress-bar { + background-color: $euiColorLightestShade; + } +} + +.lnsFieldItem__fieldPopoverPanel { + min-width: 260px; + max-width: 300px; +} + diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx index b397479bf32fd..bc3d2a135c81b 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/datapanel.tsx @@ -534,11 +534,14 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ const overallField = fieldByName[field.name]; return ( ); })} diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx index 0afc769688218..4d3d1d378c328 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/field_item.tsx @@ -4,17 +4,62 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; +import React, { useState } from 'react'; +import DateMath from '@elastic/datemath'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiProgress, + EuiPopover, + EuiLoadingSpinner, + EuiKeyboardAccessible, + EuiText, + EuiToolTip, + EuiButtonGroup, + EuiPopoverFooter, + EuiPopoverTitle, +} from '@elastic/eui'; +import { + Chart, + Axis, + getAxisId, + getSpecId, + BarSeries, + Position, + ScaleType, + Settings, + DataSeriesColorsValues, + TooltipType, + niceTimeFormatter, +} from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { toElasticsearchQuery } from '@kbn/es-query'; +import { Query } from 'src/plugins/data/common'; +// @ts-ignore +import { fieldFormats } from '../../../../../../src/legacy/ui/public/registry/field_formats'; import { IndexPattern, IndexPatternField, DraggedField } from './indexpattern'; import { DragDrop } from '../drag_drop'; -import { FieldIcon } from './field_icon'; -import { DataType } from '..'; +import { FieldIcon, getColorForDataType } from './field_icon'; +import { DatasourceDataPanelProps, DataType } from '../types'; +import { BucketedAggregation, FieldStatsResponse } from '../../common'; export interface FieldItemProps { + core: DatasourceDataPanelProps['core']; field: IndexPatternField; indexPattern: IndexPattern; highlight?: string; exists: boolean; + query: Query; + dateRange: DatasourceDataPanelProps['dateRange']; +} + +interface State { + isLoading: boolean; + totalDocuments?: number; + sampledDocuments?: number; + sampledValues?: number; + histogram?: BucketedAggregation; + topValues?: BucketedAggregation; } function wrapOnDot(str?: string) { @@ -24,7 +69,15 @@ function wrapOnDot(str?: string) { return str ? str.replace(/\./g, '.\u200B') : ''; } -export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemProps) { +export function FieldItem(props: FieldItemProps) { + const { core, field, indexPattern, highlight, exists, query, dateRange } = props; + + const [infoIsOpen, setOpen] = useState(false); + + const [state, setState] = useState({ + isLoading: false, + }); + const wrappableName = wrapOnDot(field.name)!; const wrappableHighlight = wrapOnDot(highlight); const highlightIndex = wrappableHighlight @@ -41,22 +94,407 @@ export function FieldItem({ field, indexPattern, highlight, exists }: FieldItemP ); + function fetchData() { + if ( + state.isLoading || + (field.type !== 'number' && + field.type !== 'string' && + field.type !== 'date' && + field.type !== 'boolean') + ) { + return; + } + + setState(s => ({ ...s, isLoading: true })); + + core.http + .post(`/api/lens/index_stats/${indexPattern.title}/field`, { + body: JSON.stringify({ + query: toElasticsearchQuery(query, indexPattern), + fromDate: dateRange.fromDate, + toDate: dateRange.toDate, + timeFieldName: indexPattern.timeFieldName, + field, + }), + }) + .then((results: FieldStatsResponse) => { + setState(s => ({ + ...s, + isLoading: false, + totalDocuments: results.totalDocuments, + sampledDocuments: results.sampledDocuments, + sampledValues: results.sampledValues, + histogram: results.histogram, + topValues: results.topValues, + })); + }) + .catch(() => { + setState(s => ({ ...s, isLoading: false })); + }); + } + + function togglePopover() { + setOpen(!infoIsOpen); + if (!infoIsOpen) { + fetchData(); + } + } + return ( - ('.application') || undefined} + button={ + + +
{ + togglePopover(); + }} + onKeyPress={event => { + if (event.key === 'ENTER') { + togglePopover(); + } + }} + title={i18n.translate('xpack.lens.indexPattern.fieldStatsButton', { + defaultMessage: 'Click or Enter for more information about {fieldName}', + values: { fieldName: field.name }, + })} + aria-label={i18n.translate('xpack.lens.indexPattern.fieldStatsButton', { + defaultMessage: 'Click or Enter for more information about {fieldName}', + values: { fieldName: field.name }, + })} + > + + + + {wrappableHighlightableFieldName} + +
+ + + } + isOpen={infoIsOpen} + closePopover={() => setOpen(false)} + anchorPosition="rightUp" + panelClassName="lnsFieldItem__fieldPopoverPanel" > -
- + + + ); +} + +function FieldItemPopoverContents(props: State & FieldItemProps) { + const { histogram, topValues, indexPattern, field, dateRange, core, sampledValues } = props; + + if (props.isLoading) { + return ; + } else if ( + (!props.histogram || props.histogram.buckets.length === 0) && + (!props.topValues || props.topValues.buckets.length === 0) + ) { + return ( + + {i18n.translate('xpack.lens.indexPattern.fieldStatsNoData', { + defaultMessage: 'No data to display', + })} + + ); + } + + let histogramDefault = !!props.histogram; + + const totalValuesCount = + topValues && topValues.buckets.reduce((prev, bucket) => bucket.count + prev, 0); + const otherCount = sampledValues && totalValuesCount ? sampledValues - totalValuesCount : 0; + + if ( + totalValuesCount && + histogram && + histogram.buckets.length && + topValues && + topValues.buckets.length + ) { + // Default to histogram when top values are less than 10% of total + histogramDefault = otherCount / totalValuesCount > 0.9; + } + + const [showingHistogram, setShowingHistogram] = useState(histogramDefault); + + let formatter: { convert: (data: unknown) => string }; + if (indexPattern.fieldFormatMap && indexPattern.fieldFormatMap[field.name]) { + const FormatType = fieldFormats.getType(indexPattern.fieldFormatMap[field.name].id); + if (FormatType) { + formatter = new FormatType( + indexPattern.fieldFormatMap[field.name].params, + core.uiSettings.get.bind(core.uiSettings) + ); + } else { + formatter = { convert: (data: unknown) => JSON.stringify(data) }; + } + } else { + formatter = fieldFormats.getDefaultInstance(field.type, field.esTypes); + } + + const euiButtonColor = + field.type === 'string' ? 'accent' : field.type === 'number' ? 'secondary' : 'primary'; + const euiTextColor = + field.type === 'string' ? 'accent' : field.type === 'number' ? 'secondary' : 'default'; + + const fromDate = DateMath.parse(dateRange.fromDate); + const toDate = DateMath.parse(dateRange.toDate); - - {wrappableHighlightableFieldName} - + let title = <>; + + if (histogram && histogram.buckets.length && topValues && topValues.buckets.length) { + title = ( + { + setShowingHistogram(optionId === 'histogram'); + }} + idSelected={showingHistogram ? 'histogram' : 'topValues'} + /> + ); + } else if (field.type === 'date') { + title = ( + <> + {i18n.translate('xpack.lens.indexPattern.fieldTimeDistributionLabel', { + defaultMessage: 'Time distribution', + })} + + ); + } else if (topValues && topValues.buckets.length) { + title = ( + <> + {i18n.translate('xpack.lens.indexPattern.fieldTopValuesLabel', { + defaultMessage: 'Top Values', + })} + + ); + } + + function wrapInPopover(el: React.ReactElement) { + return ( + <> + {title ? {title} : <>} + {el} + + {props.totalDocuments ? ( + + + {props.sampledDocuments && ( + <> + {i18n.translate('xpack.lens.indexPattern.percentageOfLabel', { + defaultMessage: '{percentage}% of', + values: { + percentage: Math.round((props.sampledDocuments / props.totalDocuments) * 100), + }, + })} + + )}{' '} + + {fieldFormats + .getDefaultInstance('number', ['integer']) + .convert(props.totalDocuments)} + {' '} + {i18n.translate('xpack.lens.indexPattern.ofDocumentsLabel', { + defaultMessage: 'documents', + })} + + + ) : ( + <> + )} + + ); + } + + if (histogram && histogram.buckets.length) { + const specId = getSpecId( + i18n.translate('xpack.lens.indexPattern.fieldStatsCountLabel', { + defaultMessage: 'Count', + }) + ); + const expectedColor = getColorForDataType(field.type); + const colors: DataSeriesColorsValues = { + colorValues: [], + specId, + }; + + const seriesColors = new Map([[colors, expectedColor]]); + + if (field.type === 'date') { + return wrapInPopover( + + + + + + + + ); + } else if (showingHistogram || !topValues || !topValues.buckets.length) { + return wrapInPopover( + + + + formatter.convert(d)} + /> + + + + ); + } + } + + if (props.topValues && props.topValues.buckets.length) { + return wrapInPopover( +
+ {props.topValues.buckets.map(topValue => { + const formatted = formatter.convert(topValue.key); + return ( +
+ + + {formatted === '' ? ( + + + {i18n.translate('xpack.lens.indexPattern.fieldPanelEmptyStringValue', { + defaultMessage: 'Empty string', + })} + + + ) : ( + + + {formatted} + + + )} + + + + {Math.round((topValue.count / props.sampledValues!) * 100)}% + + + + + +
+ ); + })} + {otherCount ? ( + <> + + + + {i18n.translate('xpack.lens.indexPattern.otherDocsLabel', { + defaultMessage: 'Other', + })} + + + + + + {Math.round((otherCount / props.sampledValues!) * 100)}% + + + + + + + ) : ( + <> + )}
- - ); + ); + } + return <>; } diff --git a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss index dcc579dd05ec6..733a30858dab7 100644 --- a/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss +++ b/x-pack/legacy/plugins/lens/public/indexpattern_plugin/indexpattern.scss @@ -1,3 +1,4 @@ +@import './field_itm'; @import './dimension_panel/index'; @@ -52,7 +53,6 @@ @include euiFontSizeS; background: $euiColorEmptyShade; border-radius: $euiBorderRadius; - padding: $euiSizeS; margin-bottom: $euiSizeXS; transition: box-shadow $euiAnimSpeedFast $euiAnimSlightResistance; @@ -70,3 +70,8 @@ .lnsFieldListPanel__fieldName { margin-left: $euiSizeXS; } + +.lnsFieldListPanel__fieldInfo { + padding: $euiSizeS; + font-weight: $euiFontWeightMedium; +} diff --git a/x-pack/legacy/plugins/lens/server/routes/field_stats.ts b/x-pack/legacy/plugins/lens/server/routes/field_stats.ts new file mode 100644 index 0000000000000..a57811362c6cf --- /dev/null +++ b/x-pack/legacy/plugins/lens/server/routes/field_stats.ts @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Boom from 'boom'; +import DateMath from '@elastic/datemath'; +import { schema } from '@kbn/config-schema'; +import { AggregationSearchResponse } from 'elasticsearch'; +import { CoreSetup } from 'src/core/server'; +import { FieldStatsResponse } from '../../common'; + +const SHARD_SIZE = 5000; + +export async function initFieldsRoute(setup: CoreSetup) { + const router = setup.http.createRouter(); + router.post( + { + path: '/index_stats/{indexPatternTitle}/field', + validate: { + params: schema.object({ + indexPatternTitle: schema.string(), + }), + body: schema.object( + { + query: schema.object({}, { allowUnknowns: true }), + fromDate: schema.string(), + toDate: schema.string(), + timeFieldName: schema.string(), + field: schema.object( + { + name: schema.string(), + type: schema.string(), + esTypes: schema.maybe(schema.arrayOf(schema.string())), + }, + { allowUnknowns: true } + ), + }, + { allowUnknowns: true } + ), + }, + }, + async (context, req, res) => { + const requestClient = context.core.elasticsearch.dataClient; + const { fromDate, toDate, timeFieldName, field, query } = req.body; + + try { + const filters = { + bool: { + filter: [ + { + range: { + [timeFieldName]: { + gte: fromDate, + lte: toDate, + }, + }, + }, + query, + ], + }, + }; + + const search = (aggs: unknown) => + requestClient.callAsCurrentUser('search', { + index: req.params.indexPatternTitle, + body: { + query: filters, + aggs, + }, + // The hits total changed in 7.0 from number to object, unless this flag is set + // this is a workaround for elasticsearch response types that are from 6.x + restTotalHitsAsInt: true, + size: 0, + }); + + if (field.type === 'number') { + return res.ok({ + body: await getNumberHistogram(search, field), + }); + } else if (field.type === 'string') { + return res.ok({ + body: await getStringSamples(search, field), + }); + } else if (field.type === 'date') { + return res.ok({ + body: await getDateHistogram(search, field, { fromDate, toDate }), + }); + } else if (field.type === 'boolean') { + return res.ok({ + body: await getStringSamples(search, field), + }); + } + + return res.ok({}); + } catch (e) { + if (e.status === 404) { + return res.notFound(); + } + if (e.isBoom) { + if (e.output.statusCode === 404) { + return res.notFound(); + } + return res.internalError(e.output.message); + } else { + return res.internalError({ + body: Boom.internal(e.message || e.name), + }); + } + } + } + ); +} + +export async function getNumberHistogram( + aggSearchWithBody: (body: unknown) => Promise, + field: { name: string; type: string; esTypes?: string[] } +): Promise { + const searchBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + min_value: { + min: { field: field.name }, + }, + max_value: { + max: { field: field.name }, + }, + sample_count: { value_count: { field: field.name } }, + top_values: { + terms: { field: field.name, size: 10 }, + }, + }, + }, + }; + + const minMaxResult = (await aggSearchWithBody(searchBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof searchBody } } + >; + + const minValue = minMaxResult.aggregations!.sample.min_value.value; + const maxValue = minMaxResult.aggregations!.sample.max_value.value; + const terms = minMaxResult.aggregations!.sample.top_values; + const topValuesBuckets = { + buckets: terms.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }; + + let histogramInterval = (maxValue! - minValue!) / 10; + + if (Number.isInteger(minValue!) && Number.isInteger(maxValue!)) { + histogramInterval = Math.ceil(histogramInterval); + } + + if (histogramInterval === 0) { + return { + totalDocuments: minMaxResult.hits.total, + sampledValues: minMaxResult.aggregations!.sample.sample_count.value!, + sampledDocuments: minMaxResult.aggregations!.sample.doc_count, + topValues: topValuesBuckets, + histogram: { buckets: [] }, + }; + } + + const histogramBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + histo: { + histogram: { + field: field.name, + interval: histogramInterval, + }, + }, + }, + }, + }; + const histogramResult = (await aggSearchWithBody(histogramBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof histogramBody } } + >; + + return { + totalDocuments: minMaxResult.hits.total, + sampledDocuments: minMaxResult.aggregations!.sample.doc_count, + sampledValues: minMaxResult.aggregations!.sample.sample_count.value!, + histogram: { + buckets: histogramResult.aggregations!.sample.histo.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + topValues: topValuesBuckets, + }; +} + +export async function getStringSamples( + aggSearchWithBody: (body: unknown) => unknown, + field: { name: string; type: string } +): Promise { + const topValuesBody = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { + sample_count: { value_count: { field: field.name } }, + top_values: { + terms: { field: field.name, size: 10 }, + }, + }, + }, + }; + const topValuesResult = (await aggSearchWithBody(topValuesBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof topValuesBody } } + >; + + return { + totalDocuments: topValuesResult.hits.total, + sampledDocuments: topValuesResult.aggregations!.sample.doc_count, + sampledValues: topValuesResult.aggregations!.sample.sample_count.value!, + topValues: { + buckets: topValuesResult.aggregations!.sample.top_values.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + }; +} + +// This one is not sampled so that it returns the full date range +export async function getDateHistogram( + aggSearchWithBody: (body: unknown) => unknown, + field: { name: string; type: string }, + range: { fromDate: string; toDate: string } +): Promise { + const fromDate = DateMath.parse(range.fromDate); + const toDate = DateMath.parse(range.toDate); + if (!fromDate) { + throw Error('Invalid fromDate value'); + } + if (!toDate) { + throw Error('Invalid toDate value'); + } + + const interval = Math.round((toDate.valueOf() - fromDate.valueOf()) / 10); + if (interval < 1) { + return { + totalDocuments: 0, + histogram: { buckets: [] }, + }; + } + + // TODO: Respect rollup intervals + const fixedInterval = `${interval}ms`; + + const histogramBody = { + histo: { date_histogram: { field: field.name, fixed_interval: fixedInterval } }, + }; + const results = (await aggSearchWithBody(histogramBody)) as AggregationSearchResponse< + unknown, + { body: { aggs: typeof histogramBody } } + >; + + return { + totalDocuments: results.hits.total, + histogram: { + buckets: results.aggregations!.histo.buckets.map(bucket => ({ + count: bucket.doc_count, + key: bucket.key, + })), + }, + }; +} diff --git a/x-pack/legacy/plugins/lens/server/routes/index.ts b/x-pack/legacy/plugins/lens/server/routes/index.ts index 9a957765cc87d..c5f882c8e5714 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index.ts @@ -6,7 +6,9 @@ import { CoreSetup } from 'src/core/server'; import { initStatsRoute } from './index_stats'; +import { initFieldsRoute } from './field_stats'; export function setupRoutes(setup: CoreSetup) { initStatsRoute(setup); + initFieldsRoute(setup); } diff --git a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts index aeb213e356786..d746de0a2878a 100644 --- a/x-pack/legacy/plugins/lens/server/routes/index_stats.ts +++ b/x-pack/legacy/plugins/lens/server/routes/index_stats.ts @@ -73,6 +73,7 @@ export async function initStatsRoute(setup: CoreSetup) { }, ], }, + // TODO: Add script_fields here once saved objects are available on the server }, size, }, @@ -85,8 +86,14 @@ export async function initStatsRoute(setup: CoreSetup) { } return res.ok({ body: {} }); } catch (e) { + if (e.status === 404) { + return res.notFound(); + } if (e.isBoom) { - return res.internalError(e); + if (e.output.statusCode === 404) { + return res.notFound(); + } + return res.internalError(e.output.message); } else { return res.internalError({ body: Boom.internal(e.message || e.name), diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts new file mode 100644 index 0000000000000..9eba9392c4f7f --- /dev/null +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +const TEST_START_TIME = '2015-09-19T06:31:44.000'; +const TEST_END_TIME = '2015-09-23T18:31:44.000'; +const COMMON_HEADERS = { + 'kbn-xsrf': 'some-xsrf-token', +}; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertest'); + + describe('index stats apis', () => { + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.loadIfNeeded('visualize/default'); + }); + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('visualize/default'); + }); + + describe('field distribution', () => { + it('should return a 404 for missing index patterns', async () => { + await supertest + .post('/api/lens/index_stats/logstash/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'bytes', + type: 'number', + }, + }) + .expect(404); + }); + + it('should return an auto histogram for numbers and top values', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'bytes', + type: 'number', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + sampledDocuments: 4633, + sampledValues: 4633, + histogram: { + buckets: [ + { + count: 705, + key: 0, + }, + { + count: 898, + key: 1999, + }, + { + count: 885, + key: 3998, + }, + { + count: 970, + key: 5997, + }, + { + count: 939, + key: 7996, + }, + { + count: 44, + key: 9995, + }, + { + count: 43, + key: 11994, + }, + { + count: 43, + key: 13993, + }, + { + count: 57, + key: 15992, + }, + { + count: 49, + key: 17991, + }, + ], + }, + topValues: { + buckets: [ + { + count: 147, + key: 0, + }, + { + count: 5, + key: 3954, + }, + { + count: 5, + key: 6497, + }, + { + count: 4, + key: 1840, + }, + { + count: 4, + key: 4206, + }, + { + count: 4, + key: 4328, + }, + { + count: 4, + key: 4669, + }, + { + count: 4, + key: 5846, + }, + { + count: 4, + key: 5863, + }, + { + count: 4, + key: 6631, + }, + ], + }, + }); + }); + + it('should return an auto histogram for dates', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: '@timestamp', + type: 'date', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + histogram: { + buckets: [ + { + count: 1161, + key: 1442875680000, + }, + { + count: 3420, + key: 1442914560000, + }, + { + count: 52, + key: 1442953440000, + }, + ], + }, + }); + }); + + it('should return top values for strings', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/logstash-2015.09.22/field') + .set(COMMON_HEADERS) + .send({ + query: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + field: { + name: 'geo.src', + type: 'string', + }, + }) + .expect(200); + + expect(body).to.eql({ + totalDocuments: 4633, + sampledDocuments: 4633, + sampledValues: 4633, + topValues: { + buckets: [ + { + count: 832, + key: 'CN', + }, + { + count: 804, + key: 'IN', + }, + { + count: 425, + key: 'US', + }, + { + count: 158, + key: 'ID', + }, + { + count: 143, + key: 'BR', + }, + { + count: 116, + key: 'PK', + }, + { + count: 106, + key: 'BD', + }, + { + count: 94, + key: 'NG', + }, + { + count: 84, + key: 'RU', + }, + { + count: 73, + key: 'JP', + }, + ], + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/lens/index.ts b/x-pack/test/api_integration/apis/lens/index.ts index 9827eadb1278b..d8c02db99b10a 100644 --- a/x-pack/test/api_integration/apis/lens/index.ts +++ b/x-pack/test/api_integration/apis/lens/index.ts @@ -9,5 +9,6 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function lensApiIntegrationTests({ loadTestFile }: FtrProviderContext) { describe('Lens', () => { loadTestFile(require.resolve('./index_stats')); + loadTestFile(require.resolve('./field_stats')); }); } diff --git a/x-pack/test/api_integration/apis/lens/index_stats.ts b/x-pack/test/api_integration/apis/lens/index_stats.ts index 8dc181fa9b601..17ab6a813e480 100644 --- a/x-pack/test/api_integration/apis/lens/index_stats.ts +++ b/x-pack/test/api_integration/apis/lens/index_stats.ts @@ -105,6 +105,20 @@ export default ({ getService }: FtrProviderContext) => { expect(Object.keys(body)).to.eql(fieldsWithData.map(field => field.name)); }); + + it('should throw a 404 for a non-existent index', async () => { + await supertest + .post('/api/lens/index_stats/fake') + .set(COMMON_HEADERS) + .send({ + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + timeFieldName: '@timestamp', + size: 500, + fields: [], + }) + .expect(404); + }); }); }); };