diff --git a/src/plugins/data/common/search/aggs/agg_types.ts b/src/plugins/data/common/search/aggs/agg_types.ts index 43e3df6100a75..dc1063f7f3b28 100644 --- a/src/plugins/data/common/search/aggs/agg_types.ts +++ b/src/plugins/data/common/search/aggs/agg_types.ts @@ -69,6 +69,7 @@ export const getAggTypes = () => ({ { name: BUCKET_TYPES.GEOHASH_GRID, fn: buckets.getGeoHashBucketAgg }, { name: BUCKET_TYPES.GEOTILE_GRID, fn: buckets.getGeoTitleBucketAgg }, { name: BUCKET_TYPES.SAMPLER, fn: buckets.getSamplerBucketAgg }, + { name: BUCKET_TYPES.RANDOM_SAMPLER, fn: buckets.getRandomSamplerBucketAgg }, { name: BUCKET_TYPES.DIVERSIFIED_SAMPLER, fn: buckets.getDiversifiedSamplerBucketAgg }, ], }); @@ -90,6 +91,7 @@ export const getAggTypesFunctions = () => [ buckets.aggMultiTerms, buckets.aggRareTerms, buckets.aggSampler, + buckets.aggRandomSampler, buckets.aggDiversifiedSampler, metrics.aggAvg, metrics.aggBucketAvg, diff --git a/src/plugins/data/common/search/aggs/aggs_service.test.ts b/src/plugins/data/common/search/aggs/aggs_service.test.ts index f0425e460ae0f..6563fe30c81c8 100644 --- a/src/plugins/data/common/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/common/search/aggs/aggs_service.test.ts @@ -69,6 +69,7 @@ describe('Aggs service', () => { "geotile_grid", "sampler", "diversified_sampler", + "random_sampler", "foo", ] `); @@ -123,6 +124,7 @@ describe('Aggs service', () => { "geotile_grid", "sampler", "diversified_sampler", + "random_sampler", ] `); expect(bStart.types.getAll().metrics.map((t) => t.name)).toMatchInlineSnapshot(` diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index cc49c896bdfe5..770974f8f4ac7 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -67,6 +67,10 @@ const getAggResultBuckets = ( const aggKey = keys(responseAgg)[aggId]; const aggConfig = find(aggConfigs.aggs, (agg) => agg.id === aggKey); if (aggConfig) { + if (keyParts[i] === '_no_bucket_') { + responseAgg = aggById; + break; + } const aggResultBucket = find(aggById.buckets, (bucket, bucketObjKey) => { const bucketKey = aggConfig .getKey(bucket, isNumber(bucketObjKey) ? undefined : bucketObjKey) @@ -172,16 +176,27 @@ export const buildOtherBucketAgg = ( filters: any[], key: string ) => { + const agg = aggregations[aggId]; + const newAggIndex = aggIndex + 1; + const newAgg = bucketAggs[newAggIndex]; + const currentAgg = bucketAggs[aggIndex]; + if (agg && !Array.isArray(agg?.buckets)) { + // there is no buckets array, assum an agg which is not reflected in the response and recurse + walkBucketTree( + newAggIndex, + agg, + newAgg.id, + filters, + `${key}${OTHER_BUCKET_SEPARATOR}_no_bucket_` + ); + return; + } // make sure there are actually results for the buckets - if (aggregations[aggId]?.buckets.length < 1) { + if (agg?.buckets.length < 1) { noAggBucketResults = true; return; } - const agg = aggregations[aggId]; - const newAggIndex = aggIndex + 1; - const newAgg = bucketAggs[newAggIndex]; - const currentAgg = bucketAggs[aggIndex]; if (aggIndex === index && agg && agg.sum_other_doc_count > 0) { exhaustiveBuckets = false; } diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts index fcfbb432e3055..33769ddbe59b2 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts @@ -22,5 +22,6 @@ export enum BUCKET_TYPES { GEOTILE_GRID = 'geotile_grid', DATE_HISTOGRAM = 'date_histogram', SAMPLER = 'sampler', + RANDOM_SAMPLER = 'random_sampler', DIVERSIFIED_SAMPLER = 'diversified_sampler', } diff --git a/src/plugins/data/common/search/aggs/buckets/index.ts b/src/plugins/data/common/search/aggs/buckets/index.ts index 000dcd5382b56..bb5b8f61b2f27 100644 --- a/src/plugins/data/common/search/aggs/buckets/index.ts +++ b/src/plugins/data/common/search/aggs/buckets/index.ts @@ -48,4 +48,6 @@ export * from './sampler_fn'; export * from './sampler'; export * from './diversified_sampler_fn'; export * from './diversified_sampler'; +export * from './random_sampler_fn'; +export * from './random_sampler'; export { SHARD_DELAY_AGG_NAME } from './shard_delay'; diff --git a/src/plugins/data/common/search/aggs/buckets/random_sampler.ts b/src/plugins/data/common/search/aggs/buckets/random_sampler.ts new file mode 100644 index 0000000000000..00756f1f03919 --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/random_sampler.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { BucketAggType } from './bucket_agg_type'; +import { BaseAggParams } from '../types'; +import { aggRandomSamplerFnName } from './random_sampler_fn'; + +export const RANDOM_SAMPLER_AGG_NAME = 'random_sampler'; + +const title = i18n.translate('data.search.aggs.buckets.randomSamplerTitle', { + defaultMessage: 'Random Sampler', + description: 'Random sampler aggregation title', +}); + +export interface AggParamsRandomSampler extends BaseAggParams { + /** + * The sampling probability + */ + probability: number; +} + +/** + * A filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents. + */ +export const getRandomSamplerBucketAgg = () => + new BucketAggType({ + name: RANDOM_SAMPLER_AGG_NAME, + title, + customLabels: false, + expressionName: aggRandomSamplerFnName, + params: [ + { + name: 'probability', + type: 'number', + }, + ], + }); diff --git a/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.test.ts b/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.test.ts new file mode 100644 index 0000000000000..b14e9b1631896 --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { functionWrapper } from '../test_helpers'; +import { aggRandomSampler } from './random_sampler_fn'; + +describe('aggRandomSampler', () => { + const fn = functionWrapper(aggRandomSampler()); + + test('includes params when they are provided', () => { + const actual = fn({ + id: 'random_sampler', + schema: 'bucket', + probability: 0.1, + }); + + expect(actual.value).toMatchInlineSnapshot(` + Object { + "enabled": true, + "id": "random_sampler", + "params": Object { + "probability": 0.1, + }, + "schema": "bucket", + "type": "random_sampler", + } + `); + }); +}); diff --git a/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.ts b/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.ts new file mode 100644 index 0000000000000..50d0243a27c11 --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/random_sampler_fn.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common'; +import { AggExpressionFunctionArgs, AggExpressionType, BUCKET_TYPES } from '..'; +import { RANDOM_SAMPLER_AGG_NAME } from './random_sampler'; + +export const aggRandomSamplerFnName = 'aggRandomSampler'; + +type Input = any; +type Arguments = AggExpressionFunctionArgs; + +type Output = AggExpressionType; +type FunctionDefinition = ExpressionFunctionDefinition< + typeof aggRandomSamplerFnName, + Input, + Arguments, + Output +>; + +export const aggRandomSampler = (): FunctionDefinition => ({ + name: aggRandomSamplerFnName, + help: i18n.translate('data.search.aggs.function.buckets.randomSampler.help', { + defaultMessage: 'Generates a serialized agg config for a Random sampler agg', + }), + type: 'agg_type', + args: { + id: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.randomSampler.id.help', { + defaultMessage: 'ID for this aggregation', + }), + }, + enabled: { + types: ['boolean'], + default: true, + help: i18n.translate('data.search.aggs.buckets.randomSampler.enabled.help', { + defaultMessage: 'Specifies whether this aggregation should be enabled', + }), + }, + schema: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.randomSampler.schema.help', { + defaultMessage: 'Schema to use for this aggregation', + }), + }, + probability: { + types: ['number'], + help: i18n.translate('data.search.aggs.buckets.randomSampler.probability.help', { + defaultMessage: 'The sampling probability', + }), + }, + }, + fn: (input, args) => { + const { id, enabled, schema, ...rest } = args; + + return { + type: 'agg_type', + value: { + id, + enabled, + schema, + type: RANDOM_SAMPLER_AGG_NAME, + params: { + ...rest, + }, + }, + }; + }, +}); diff --git a/src/plugins/data/common/search/aggs/metrics/lib/sibling_pipeline_agg_helper.ts b/src/plugins/data/common/search/aggs/metrics/lib/sibling_pipeline_agg_helper.ts index 243a119847a2c..c50eb36468e9f 100644 --- a/src/plugins/data/common/search/aggs/metrics/lib/sibling_pipeline_agg_helper.ts +++ b/src/plugins/data/common/search/aggs/metrics/lib/sibling_pipeline_agg_helper.ts @@ -31,7 +31,13 @@ const metricAggFilter: string[] = [ '!filtered_metric', '!single_percentile', ]; -const bucketAggFilter: string[] = ['!filter', '!sampler', '!diversified_sampler', '!multi_terms']; +const bucketAggFilter: string[] = [ + '!filter', + '!sampler', + '!diversified_sampler', + '!random_sampler', + '!multi_terms', +]; export const siblingPipelineType = i18n.translate( 'data.search.aggs.metrics.siblingPipelineAggregationsSubtypeTitle', diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index bc35e46d8da7a..19ecd1eaa866d 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -109,8 +109,10 @@ import { AggParamsMovingAvgSerialized, AggParamsSerialDiffSerialized, AggParamsTopHitSerialized, + aggRandomSampler, } from '.'; import { AggParamsSampler } from './buckets/sampler'; +import { AggParamsRandomSampler } from './buckets/random_sampler'; import { AggParamsDiversifiedSampler } from './buckets/diversified_sampler'; import { AggParamsSignificantText } from './buckets/significant_text'; import { aggTopMetrics } from './metrics/top_metrics_fn'; @@ -185,6 +187,7 @@ interface SerializedAggParamsMapping { [BUCKET_TYPES.MULTI_TERMS]: AggParamsMultiTermsSerialized; [BUCKET_TYPES.RARE_TERMS]: AggParamsRareTerms; [BUCKET_TYPES.SAMPLER]: AggParamsSampler; + [BUCKET_TYPES.RANDOM_SAMPLER]: AggParamsRandomSampler; [BUCKET_TYPES.DIVERSIFIED_SAMPLER]: AggParamsDiversifiedSampler; [METRIC_TYPES.AVG]: AggParamsAvg; [METRIC_TYPES.CARDINALITY]: AggParamsCardinality; @@ -230,6 +233,7 @@ export interface AggParamsMapping { [BUCKET_TYPES.MULTI_TERMS]: AggParamsMultiTerms; [BUCKET_TYPES.RARE_TERMS]: AggParamsRareTerms; [BUCKET_TYPES.SAMPLER]: AggParamsSampler; + [BUCKET_TYPES.RANDOM_SAMPLER]: AggParamsRandomSampler; [BUCKET_TYPES.DIVERSIFIED_SAMPLER]: AggParamsDiversifiedSampler; [METRIC_TYPES.AVG]: AggParamsAvg; [METRIC_TYPES.CARDINALITY]: AggParamsCardinality; @@ -265,6 +269,7 @@ export interface AggFunctionsMapping { aggFilter: ReturnType; aggFilters: ReturnType; aggSignificantTerms: ReturnType; + aggRandomSampler: ReturnType; aggIpRange: ReturnType; aggDateRange: ReturnType; aggRange: ReturnType; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 3d1068ebd521f..58160c9c04ca5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -319,6 +319,11 @@ export function LayerPanel( visualizationState, updateVisualization ) || []), + ...(layerDatasource?.getSupportedActionsForLayer?.( + layerId, + layerDatasourceState, + (newState) => updateDatasource(datasourceId, newState) + ) || []), ...getSharedActions({ activeVisualization, core, @@ -333,12 +338,16 @@ export function LayerPanel( [ activeVisualization, core, + datasourceId, isOnlyLayer, isTextBasedLanguage, + layerDatasource, + layerDatasourceState, layerId, layerIndex, onCloneLayer, onRemoveLayer, + updateDatasource, updateVisualization, visualizationState, ] diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index fb31c3c1a9a71..a0652d1265e8c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useState } from 'react'; import { render } from 'react-dom'; import { I18nProvider } from '@kbn/i18n-react'; import type { CoreStart, SavedObjectReference } from '@kbn/core/public'; @@ -17,14 +17,27 @@ import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { flatten, isEqual } from 'lodash'; import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; -import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { + KibanaContextProvider, + KibanaThemeProvider, + toMountPoint, +} from '@kbn/kibana-react-plugin/public'; import { DataPublicPluginStart, ES_FIELD_TYPES } from '@kbn/data-plugin/public'; import { VisualizeFieldContext } from '@kbn/ui-actions-plugin/public'; import { ChartsPluginSetup } from '@kbn/charts-plugin/public'; import { UiActionsStart } from '@kbn/ui-actions-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; import { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; -import { EuiCallOut, EuiLink } from '@elastic/eui'; +import { + EuiBadge, + EuiButton, + EuiCallOut, + EuiLink, + EuiModalBody, + EuiModalHeader, + EuiModalHeaderTitle, + EuiRadioGroup, +} from '@elastic/eui'; import type { DatasourceDimensionEditorProps, DatasourceDimensionTriggerProps, @@ -744,6 +757,70 @@ export function getIndexPatternDatasource({ const ids = Object.values(state.layers || {}).map(({ indexPatternId }) => indexPatternId); return ids.filter((id) => !indexPatterns[id]); }, + getSupportedActionsForLayer(layerId, state, setState) { + const layer = state.layers[layerId]; + return [ + { + displayName: + layer.sampling && layer.sampling !== 1 + ? `Change random sampling (${layer.sampling}) - tech preview` + : `Enable random sampling - tech preview`, + icon: 'empty', + isCompatible: true, + execute: () => { + const SamplingModal = () => { + const [rate, setRate] = useState(layer.sampling || 1); + return ( + <> + + +

+ Random Sampling Tech preview +

+ + + +

Change the sampling probability to see how your chart is affected

+ { + setState({ + ...state, + layers: { + ...state.layers, + [layerId]: { + ...layer, + sampling: Number(e), + }, + }, + }); + setRate(Number(e)); + }} + options={[ + { label: '0.0001', id: '0.0001' }, + { label: '0.001', id: '0.001' }, + { label: '0.01', id: '0.01' }, + { label: '0.1', id: '0.1' }, + { label: '1 (No sampling)', id: '1' }, + ]} + /> + { + overlayRef.close(); + }} + > + Done + + + + ); + }; + + const overlayRef = core.overlays.openModal(toMountPoint()); + }, + }, + ]; + }, isTimeBased: (state, indexPatterns) => { if (!state) return false; const { layers } = state; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 72cb2a2ab729e..6babf4e5a1c5e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -132,6 +132,26 @@ function getExpressionForLayer( } }); + const doSample = layer.sampling && layer.sampling !== 1; + const bucketOffset = doSample ? 1 : 0; + const metricOffset = bucketOffset + (window.ELASTIC_LENS_DELAY_SECONDS ? 1 : 0); + + if (doSample) { + aggs.push( + buildExpression({ + type: 'expression', + chain: [ + buildExpressionFunction('aggRandomSampler', { + id: 'the-sampling', + enabled: true, + schema: 'bucket', + probability: layer.sampling, + }).toAst(), + ], + }) + ); + } + const orderedColumnIds = esAggEntries.map(([colId]) => colId); let esAggsIdMap: Record = {}; const aggExpressionToEsAggsIdMap: Map = new Map(); @@ -184,9 +204,9 @@ function getExpressionForLayer( }); aggs.push(expressionBuilder); - const esAggsId = window.ELASTIC_LENS_DELAY_SECONDS - ? `col-${index + (col.isBucketed ? 0 : 1)}-${aggId}` - : `col-${index}-${aggId}`; + const esAggsId = `col-${ + index + (col.isBucketed && doSample ? bucketOffset : metricOffset) + }-${aggId}`; esAggsIdMap[esAggsId] = [ { @@ -270,9 +290,7 @@ function getExpressionForLayer( matchingEsAggColumnIds.forEach((currentId) => { const currentColumn = esAggsIdMap[currentId][0]; - const aggIndex = window.ELASTIC_LENS_DELAY_SECONDS - ? counter + (currentColumn.isBucketed ? 0 : 1) - : counter; + const aggIndex = counter + (currentColumn.isBucketed ? bucketOffset : metricOffset); const newId = updatePositionIndex(currentId, aggIndex); updatedEsAggsIdMap[newId] = esAggsIdMap[currentId]; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 5ff9b183d2579..4140ea076649c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -54,6 +54,7 @@ export interface IndexPatternLayer { indexPatternId: string; // Partial columns represent the temporary invalid states incompleteColumns?: Record; + sampling?: number; } export interface IndexPatternPersistedState { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 29aff3d428690..e25f2a76bb89b 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -439,6 +439,15 @@ export interface Datasource { * Get all the used DataViews from state */ getUsedDataViews: (state: T) => string[]; + /** + * returns a list of custom actions supported by the datasource layer. + * Default actions like delete/clear are not included in this list and are managed by the editor frame + * */ + getSupportedActionsForLayer?: ( + layerId: string, + state: T, + setState: StateSetter + ) => LayerAction[]; } export interface DatasourceFixAction {