diff --git a/packages/kbn-management/cards_navigation/src/consts.tsx b/packages/kbn-management/cards_navigation/src/consts.tsx index 16e655c5510ad..6a22b9e33d620 100644 --- a/packages/kbn-management/cards_navigation/src/consts.tsx +++ b/packages/kbn-management/cards_navigation/src/consts.tsx @@ -77,7 +77,7 @@ export const appDefinitions: Record = { description: i18n.translate('management.landing.withCardNavigation.dataUsageDescription', { defaultMessage: 'View data usage and retention.', }), - icon: 'documents', + icon: 'stats', }, [AppIds.RULES]: { diff --git a/x-pack/plugins/data_usage/common/index.ts b/x-pack/plugins/data_usage/common/index.ts index 4b6f899b58d37..eb0787f53f344 100644 --- a/x-pack/plugins/data_usage/common/index.ts +++ b/x-pack/plugins/data_usage/common/index.ts @@ -11,3 +11,7 @@ export const PLUGIN_ID = 'data_usage'; export const PLUGIN_NAME = i18n.translate('xpack.dataUsage.name', { defaultMessage: 'Data Usage', }); + +export const DATA_USAGE_API_ROUTE_PREFIX = '/api/data_usage/'; +export const DATA_USAGE_METRICS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}metrics`; +export const DATA_USAGE_DATA_STREAMS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}data_streams`; diff --git a/x-pack/plugins/data_usage/common/query_client.tsx b/x-pack/plugins/data_usage/common/query_client.tsx new file mode 100644 index 0000000000000..8a64ed7a51349 --- /dev/null +++ b/x-pack/plugins/data_usage/common/query_client.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PropsWithChildren } from 'react'; +import React, { memo, useMemo } from 'react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; + +type QueryClientOptionsProp = ConstructorParameters[0]; + +/** + * Default Query Client for Data Usage. + */ +export class DataUsageQueryClient extends QueryClient { + constructor(options: QueryClientOptionsProp = {}) { + const optionsWithDefaults: QueryClientOptionsProp = { + ...options, + defaultOptions: { + ...(options.defaultOptions ?? {}), + queries: { + refetchIntervalInBackground: false, + refetchOnWindowFocus: false, + refetchOnMount: true, + keepPreviousData: true, + ...(options?.defaultOptions?.queries ?? {}), + }, + }, + }; + super(optionsWithDefaults); + } +} + +/** + * The default Data Usage Query Client. Can be imported and used from outside of React hooks + * and still benefit from ReactQuery features (like caching, etc) + * + * @see https://tanstack.com/query/v4/docs/reference/QueryClient + */ +export const dataUsageQueryClient = new DataUsageQueryClient(); + +export type ReactQueryClientProviderProps = PropsWithChildren<{ + queryClient?: DataUsageQueryClient; +}>; + +export const DataUsageReactQueryClientProvider = memo( + ({ queryClient, children }) => { + const client = useMemo(() => { + return queryClient || dataUsageQueryClient; + }, [queryClient]); + return {children}; + } +); + +DataUsageReactQueryClientProvider.displayName = 'DataUsageReactQueryClientProvider'; diff --git a/x-pack/plugins/data_usage/server/types.ts b/x-pack/plugins/data_usage/common/rest_types/data_streams.ts similarity index 51% rename from x-pack/plugins/data_usage/server/types.ts rename to x-pack/plugins/data_usage/common/rest_types/data_streams.ts index 9f43ae2d3c298..b1c02bb40854d 100644 --- a/x-pack/plugins/data_usage/server/types.ts +++ b/x-pack/plugins/data_usage/common/rest_types/data_streams.ts @@ -5,12 +5,14 @@ * 2.0. */ -/* eslint-disable @typescript-eslint/no-empty-interface*/ +import { schema } from '@kbn/config-schema'; -export interface DataUsageSetupDependencies {} - -export interface DataUsageStartDependencies {} - -export interface DataUsageServerSetup {} - -export interface DataUsageServerStart {} +export const DataStreamsResponseSchema = { + body: () => + schema.arrayOf( + schema.object({ + name: schema.string(), + storageSizeBytes: schema.number(), + }) + ), +}; diff --git a/x-pack/plugins/data_usage/common/rest_types/index.ts b/x-pack/plugins/data_usage/common/rest_types/index.ts new file mode 100644 index 0000000000000..64b5c640ebbb5 --- /dev/null +++ b/x-pack/plugins/data_usage/common/rest_types/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './usage_metrics'; +export * from './data_streams'; diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts new file mode 100644 index 0000000000000..f6c08e2caddc0 --- /dev/null +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.test.ts @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UsageMetricsRequestSchema } from './usage_metrics'; + +describe('usage_metrics schemas', () => { + it('should accept valid request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + }) + ).not.toThrow(); + }); + + it('should accept a single `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'ingest_rate', + }) + ).not.toThrow(); + }); + + it('should accept multiple `metricTypes` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['ingest_rate', 'storage_retained', 'index_rate'], + }) + ).not.toThrow(); + }); + + it('should accept a single string as `dataStreams` in request query', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'storage_retained', + dataStreams: 'data_stream_1', + }) + ).not.toThrow(); + }); + + it('should accept `dataStream` list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'], + }) + ).not.toThrow(); + }); + + it('should error if `dataStream` list is empty', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + dataStreams: [], + }) + ).toThrowError('expected value of type [string] but got [Array]'); + }); + + it('should error if `dataStream` is given an empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + dataStreams: ' ', + }) + ).toThrow('[dataStreams] must have at least one value'); + }); + + it('should error if `dataStream` is given an empty item in the list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained'], + dataStreams: ['ds_1', ' '], + }) + ).toThrow('[dataStreams] list can not contain empty values'); + }); + + it('should error if `metricTypes` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ' ', + }) + ).toThrow(); + }); + + it('should error if `metricTypes` is empty item', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: [' ', 'storage_retained'], + }) + ).toThrow('[metricTypes] list can not contain empty values'); + }); + + it('should error if `metricTypes` is not a valid value', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: 'foo', + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `metricTypes` is not a valid list', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow( + '[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' + ); + }); + + it('should error if `from` is not a valid input', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: 1010, + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[from]: expected value of type [string] but got [number]'); + }); + + it('should error if `to` is not a valid input', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: 1010, + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[to]: expected value of type [string] but got [number]'); + }); + + it('should error if `from` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: ' ', + to: new Date().toISOString(), + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[from]: Date ISO string must not be empty'); + }); + + it('should error if `to` is empty string', () => { + expect(() => + UsageMetricsRequestSchema.query.validate({ + from: new Date().toISOString(), + to: ' ', + metricTypes: ['storage_retained', 'foo'], + }) + ).toThrow('[to]: Date ISO string must not be empty'); + }); +}); diff --git a/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts new file mode 100644 index 0000000000000..f2bbdb616fc79 --- /dev/null +++ b/x-pack/plugins/data_usage/common/rest_types/usage_metrics.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +const METRIC_TYPE_VALUES = [ + 'storage_retained', + 'ingest_rate', + 'search_vcu', + 'ingest_vcu', + 'ml_vcu', + 'index_latency', + 'index_rate', + 'search_latency', + 'search_rate', +] as const; + +export type MetricTypes = (typeof METRIC_TYPE_VALUES)[number]; + +// type guard for MetricTypes +export const isMetricType = (type: string): type is MetricTypes => + METRIC_TYPE_VALUES.includes(type as MetricTypes); + +// @ts-ignore +const isValidMetricType = (value: string) => METRIC_TYPE_VALUES.includes(value); + +const DateSchema = schema.string({ + minLength: 1, + validate: (v) => (v.trim().length ? undefined : 'Date ISO string must not be empty'), +}); + +const metricTypesSchema = schema.oneOf( + // @ts-expect-error TS2769: No overload matches this call + METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType)) // Create a oneOf schema for the keys +); +export const UsageMetricsRequestSchema = { + query: schema.object({ + from: DateSchema, + to: DateSchema, + metricTypes: schema.oneOf([ + schema.arrayOf(schema.string(), { + minSize: 1, + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[metricTypes] list can not contain empty values'; + } else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + schema.string({ + validate: (v) => { + if (!v.trim().length) { + return '[metricTypes] must have at least one value'; + } else if (!isValidMetricType(v)) { + return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`; + } + }, + }), + ]), + dataStreams: schema.maybe( + schema.oneOf([ + schema.arrayOf(schema.string(), { + minSize: 1, + validate: (values) => { + if (values.map((v) => v.trim()).some((v) => !v.length)) { + return '[dataStreams] list can not contain empty values'; + } + }, + }), + schema.string({ + validate: (v) => + v.trim().length ? undefined : '[dataStreams] must have at least one value', + }), + ]) + ), + }), +}; + +export type UsageMetricsRequestSchemaQueryParams = TypeOf; + +export const UsageMetricsResponseSchema = { + body: () => + schema.object({ + metrics: schema.recordOf( + metricTypesSchema, + schema.arrayOf( + schema.object({ + name: schema.string(), + data: schema.arrayOf( + schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 }) // Each data point is an array of 2 numbers + ), + }) + ) + ), + }), +}; +export type UsageMetricsResponseSchemaBody = TypeOf; diff --git a/x-pack/plugins/data_usage/common/types.ts b/x-pack/plugins/data_usage/common/types.ts new file mode 100644 index 0000000000000..d80bae2458d09 --- /dev/null +++ b/x-pack/plugins/data_usage/common/types.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// temporary type until agreed on +export type MetricKey = 'ingestedMax' | 'retainedMax'; diff --git a/x-pack/plugins/data_usage/kibana.jsonc b/x-pack/plugins/data_usage/kibana.jsonc index 9b0f2d193925e..ffd8833351267 100644 --- a/x-pack/plugins/data_usage/kibana.jsonc +++ b/x-pack/plugins/data_usage/kibana.jsonc @@ -1,16 +1,28 @@ { "type": "plugin", "id": "@kbn/data-usage-plugin", - "owner": ["@elastic/obs-ai-assistant", "@elastic/security-solution"], + "owner": [ + "@elastic/obs-ai-assistant", + "@elastic/security-solution" + ], "plugin": { "id": "dataUsage", "server": true, "browser": true, - "configPath": ["xpack", "dataUsage"], - "requiredPlugins": ["home", "management", "features", "share"], + "configPath": [ + "xpack", + "dataUsage" + ], + "requiredPlugins": [ + "home", + "management", + "features", + "share" + ], "optionalPlugins": [], "requiredBundles": [ "kibanaReact", - ], + "data" + ] } } diff --git a/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx new file mode 100644 index 0000000000000..c7937ae149de9 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/chart_panel.tsx @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo } from 'react'; +import numeral from '@elastic/numeral'; +import { EuiFlexItem, EuiPanel, EuiTitle, useEuiTheme } from '@elastic/eui'; +import { + Chart, + Axis, + BarSeries, + Settings, + ScaleType, + niceTimeFormatter, + DARK_THEME, + LIGHT_THEME, +} from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import { LegendAction } from './legend_action'; +import { MetricTypes } from '../../../common/rest_types'; +import { MetricSeries } from '../types'; + +// TODO: Remove this when we have a title for each metric type +type ChartKey = Extract; +export const chartKeyToTitleMap: Record = { + ingest_rate: i18n.translate('xpack.dataUsage.charts.ingestedMax', { + defaultMessage: 'Data Ingested', + }), + storage_retained: i18n.translate('xpack.dataUsage.charts.retainedMax', { + defaultMessage: 'Data Retained in Storage', + }), +}; + +interface ChartPanelProps { + metricType: MetricTypes; + series: MetricSeries[]; + idx: number; + popoverOpen: string | null; + togglePopover: (streamName: string | null) => void; +} + +export const ChartPanel: React.FC = ({ + metricType, + series, + idx, + popoverOpen, + togglePopover, +}) => { + const theme = useEuiTheme(); + + const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d[0])); + + const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)]; + + const tickFormat = useMemo( + () => niceTimeFormatter([minTimestamp, maxTimestamp]), + [minTimestamp, maxTimestamp] + ); + + const renderLegendAction = useCallback( + ({ label }: { label: string }) => { + return ( + + ); + }, + [idx, popoverOpen, togglePopover] + ); + return ( + + + +
{chartKeyToTitleMap[metricType as ChartKey] || metricType}
+
+ + + {series.map((stream, streamIdx) => ( + + ))} + + + + formatBytes(d)} + /> + +
+
+ ); +}; +const formatBytes = (bytes: number) => { + return numeral(bytes).format('0.0 b'); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/charts.tsx b/x-pack/plugins/data_usage/public/app/components/charts.tsx new file mode 100644 index 0000000000000..6549f7e03830a --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/charts.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useState } from 'react'; +import { EuiFlexGroup } from '@elastic/eui'; +import { MetricsResponse } from '../types'; +import { MetricTypes } from '../../../common/rest_types'; +import { ChartPanel } from './chart_panel'; +interface ChartsProps { + data: MetricsResponse; +} + +export const Charts: React.FC = ({ data }) => { + const [popoverOpen, setPopoverOpen] = useState(null); + const togglePopover = useCallback((streamName: string | null) => { + setPopoverOpen((prev) => (prev === streamName ? null : streamName)); + }, []); + + return ( + + {Object.entries(data.metrics).map(([metricType, series], idx) => ( + + ))} + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx new file mode 100644 index 0000000000000..d6627f3d8dca2 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiListGroupItem } from '@elastic/eui'; +import { + DataQualityDetailsLocatorParams, + DATA_QUALITY_DETAILS_LOCATOR_ID, +} from '@kbn/deeplinks-observability'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; +import { useDateRangePicker } from '../hooks/use_date_picker'; + +interface DatasetQualityLinkProps { + dataStreamName: string; +} + +export const DatasetQualityLink: React.FC = React.memo( + ({ dataStreamName }) => { + const { dateRangePickerState } = useDateRangePicker(); + const { + services: { + share: { url }, + }, + } = useKibanaContextForPlugin(); + const { startDate, endDate } = dateRangePickerState; + const locator = url.locators.get( + DATA_QUALITY_DETAILS_LOCATOR_ID + ); + const onClickDataQuality = async () => { + const locatorParams: DataQualityDetailsLocatorParams = { + dataStream: dataStreamName, + timeRange: { from: startDate, to: endDate, refresh: { pause: true, value: 0 } }, + }; + if (locator) { + await locator.navigate(locatorParams); + } + }; + return ; + } +); diff --git a/x-pack/plugins/data_usage/public/app/components/date_picker.tsx b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx new file mode 100644 index 0000000000000..ca29acf8c96a6 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/date_picker.tsx @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiSuperDatePicker, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/react'; +import type { IUnifiedSearchPluginServices } from '@kbn/unified-search-plugin/public'; +import type { EuiSuperDatePickerRecentRange } from '@elastic/eui'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; + +export interface DateRangePickerValues { + autoRefreshOptions: { + enabled: boolean; + duration: number; + }; + startDate: string; + endDate: string; + recentlyUsedDateRanges: EuiSuperDatePickerRecentRange[]; +} + +interface UsageMetricsDateRangePickerProps { + dateRangePickerState: DateRangePickerValues; + isDataLoading: boolean; + onRefresh: () => void; + onRefreshChange: (evt: OnRefreshChangeProps) => void; + onTimeChange: ({ start, end }: DurationRange) => void; +} + +export const UsageMetricsDateRangePicker = memo( + ({ dateRangePickerState, isDataLoading, onRefresh, onRefreshChange, onTimeChange }) => { + const { euiTheme } = useEuiTheme(); + const kibana = useKibana(); + const { uiSettings } = kibana.services; + const [commonlyUsedRanges] = useState(() => { + return ( + uiSettings + ?.get(UI_SETTINGS.TIMEPICKER_QUICK_RANGES) + ?.map(({ from, to, display }: { from: string; to: string; display: string }) => { + return { + start: from, + end: to, + label: display, + }; + }) ?? [] + ); + }); + + return ( +
+ + + + + +
+ ); + } +); + +UsageMetricsDateRangePicker.displayName = 'UsageMetricsDateRangePicker'; diff --git a/x-pack/plugins/data_usage/public/app/components/legend_action.tsx b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx new file mode 100644 index 0000000000000..a816d1f8eadda --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/components/legend_action.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiButtonIcon, + EuiPopover, + EuiListGroup, + EuiListGroupItem, + EuiSpacer, +} from '@elastic/eui'; +import { DatasetQualityLink } from './dataset_quality_link'; +import { useKibanaContextForPlugin } from '../../utils/use_kibana'; + +interface LegendActionProps { + idx: number; + popoverOpen: string | null; + togglePopover: (streamName: string | null) => void; + label: string; +} + +export const LegendAction: React.FC = React.memo( + ({ label, idx, popoverOpen, togglePopover }) => { + const uniqueStreamName = `${idx}-${label}`; + const { + services: { + share: { + url: { locators }, + }, + application: { capabilities }, + }, + } = useKibanaContextForPlugin(); + const hasDataSetQualityFeature = !!capabilities?.data_quality; + const hasIndexManagementFeature = !!capabilities?.index_management; + + const onClickIndexManagement = useCallback(async () => { + // TODO: use proper index management locator https://github.com/elastic/kibana/issues/195083 + const dataQualityLocator = locators.get('MANAGEMENT_APP_LOCATOR'); + if (dataQualityLocator) { + await dataQualityLocator.navigate({ + sectionId: 'data', + appId: `index_management/data_streams/${label}`, + }); + } + togglePopover(null); // Close the popover after action + }, [label, locators, togglePopover]); + + const onCopyDataStreamName = useCallback(() => { + navigator.clipboard.writeText(label); + togglePopover(null); // Close popover after copying + }, [label, togglePopover]); + + return ( + + + + togglePopover(uniqueStreamName)} + /> + + + } + isOpen={popoverOpen === uniqueStreamName} + closePopover={() => togglePopover(null)} + anchorPosition="downRight" + > + + + + + {hasIndexManagementFeature && ( + + )} + {hasDataSetQualityFeature && } + + + + ); + } +); diff --git a/x-pack/plugins/data_usage/public/app/data_usage.tsx b/x-pack/plugins/data_usage/public/app/data_usage.tsx new file mode 100644 index 0000000000000..c32f86d68b5bf --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/data_usage.tsx @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { + EuiTitle, + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiLoadingElastic, + EuiPageSection, + EuiText, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { i18n } from '@kbn/i18n'; +import { UsageMetricsRequestSchemaQueryParams } from '../../common/rest_types'; +import { Charts } from './components/charts'; +import { UsageMetricsDateRangePicker } from './components/date_picker'; +import { useBreadcrumbs } from '../utils/use_breadcrumbs'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; +import { PLUGIN_NAME } from '../../common'; +import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics'; +import { DEFAULT_DATE_RANGE_OPTIONS, useDateRangePicker } from './hooks/use_date_picker'; +import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params'; +import { MetricsResponse } from './types'; + +export const DataUsage = () => { + const { + services: { chrome, appParams }, + } = useKibanaContextForPlugin(); + + const { + metricTypes: metricTypesFromUrl, + dataStreams: dataStreamsFromUrl, + startDate: startDateFromUrl, + endDate: endDateFromUrl, + setUrlMetricTypesFilter, + setUrlDateRangeFilter, + } = useDataUsageMetricsUrlParams(); + + const [queryParams, setQueryParams] = useState({ + metricTypes: ['storage_retained', 'ingest_rate'], + dataStreams: [], + from: DEFAULT_DATE_RANGE_OPTIONS.startDate, + to: DEFAULT_DATE_RANGE_OPTIONS.endDate, + }); + + useEffect(() => { + if (!metricTypesFromUrl) { + setUrlMetricTypesFilter( + typeof queryParams.metricTypes !== 'string' + ? queryParams.metricTypes.join(',') + : queryParams.metricTypes + ); + } + if (!startDateFromUrl || !endDateFromUrl) { + setUrlDateRangeFilter({ startDate: queryParams.from, endDate: queryParams.to }); + } + }, [ + endDateFromUrl, + metricTypesFromUrl, + queryParams.from, + queryParams.metricTypes, + queryParams.to, + setUrlDateRangeFilter, + setUrlMetricTypesFilter, + startDateFromUrl, + ]); + + useEffect(() => { + setQueryParams((prevState) => ({ + ...prevState, + metricTypes: metricTypesFromUrl?.length ? metricTypesFromUrl : prevState.metricTypes, + dataStreams: dataStreamsFromUrl?.length ? dataStreamsFromUrl : prevState.dataStreams, + })); + }, [metricTypesFromUrl, dataStreamsFromUrl]); + + const { dateRangePickerState, onRefreshChange, onTimeChange } = useDateRangePicker(); + + const { + error, + data, + isFetching, + isFetched, + refetch: refetchDataUsageMetrics, + } = useGetDataUsageMetrics( + { + ...queryParams, + from: dateRangePickerState.startDate, + to: dateRangePickerState.endDate, + }, + { + retry: false, + } + ); + + const onRefresh = useCallback(() => { + refetchDataUsageMetrics(); + }, [refetchDataUsageMetrics]); + + useBreadcrumbs([{ text: PLUGIN_NAME }], appParams, chrome); + + // TODO: show a toast? + if (!isFetching && error?.body) { + return
{error.body.message}
; + } + + return ( + <> + +

+ {i18n.translate('xpack.dataUsage.pageTitle', { + defaultMessage: 'Data Usage', + })} +

+
+ + + + + + + + + + + + + + {isFetched && data ? : } + + + ); +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx new file mode 100644 index 0000000000000..0e03da5d9adbd --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; +import { MetricTypes, isMetricType } from '../../../common/rest_types'; +import { useUrlParams } from '../../hooks/use_url_params'; +import { DEFAULT_DATE_RANGE_OPTIONS } from './use_date_picker'; + +interface UrlParamsDataUsageMetricsFilters { + metricTypes: string; + dataStreams: string; + startDate: string; + endDate: string; +} + +interface DataUsageMetricsFiltersFromUrlParams { + metricTypes?: MetricTypes[]; + dataStreams?: string[]; + startDate?: string; + endDate?: string; + setUrlDataStreamsFilter: (dataStreams: UrlParamsDataUsageMetricsFilters['dataStreams']) => void; + setUrlDateRangeFilter: ({ startDate, endDate }: { startDate: string; endDate: string }) => void; + setUrlMetricTypesFilter: (metricTypes: UrlParamsDataUsageMetricsFilters['metricTypes']) => void; +} + +type FiltersFromUrl = Pick< + DataUsageMetricsFiltersFromUrlParams, + 'metricTypes' | 'dataStreams' | 'startDate' | 'endDate' +>; + +export const getDataUsageMetricsFiltersFromUrlParams = ( + urlParams: Partial +): FiltersFromUrl => { + const dataUsageMetricsFilters: FiltersFromUrl = { + metricTypes: [], + dataStreams: [], + startDate: DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: DEFAULT_DATE_RANGE_OPTIONS.endDate, + }; + + const urlMetricTypes = urlParams.metricTypes + ? String(urlParams.metricTypes) + .split(',') + .reduce((acc, curr) => { + if (isMetricType(curr)) { + acc.push(curr); + } + return acc.sort(); + }, []) + : []; + + const urlDataStreams = urlParams.dataStreams + ? String(urlParams.dataStreams).split(',').sort() + : []; + + dataUsageMetricsFilters.metricTypes = urlMetricTypes.length ? urlMetricTypes : undefined; + dataUsageMetricsFilters.dataStreams = urlDataStreams.length ? urlDataStreams : undefined; + dataUsageMetricsFilters.startDate = urlParams.startDate ? String(urlParams.startDate) : undefined; + dataUsageMetricsFilters.endDate = urlParams.endDate ? String(urlParams.endDate) : undefined; + + return dataUsageMetricsFilters; +}; + +export const useDataUsageMetricsUrlParams = (): DataUsageMetricsFiltersFromUrlParams => { + const location = useLocation(); + const history = useHistory(); + const { urlParams, toUrlParams } = useUrlParams(); + + const getUrlDataUsageMetricsFilters: FiltersFromUrl = useMemo( + () => getDataUsageMetricsFiltersFromUrlParams(urlParams), + [urlParams] + ); + const [dataUsageMetricsFilters, setDataUsageMetricsFilters] = useState( + getUrlDataUsageMetricsFilters + ); + + const setUrlMetricTypesFilter = useCallback( + (metricTypes: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + metricTypes: metricTypes.length ? metricTypes : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDataStreamsFilter = useCallback( + (dataStreams: string) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + dataStreams: dataStreams.length ? dataStreams : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + const setUrlDateRangeFilter = useCallback( + ({ startDate, endDate }: { startDate: string; endDate: string }) => { + history.push({ + ...location, + search: toUrlParams({ + ...urlParams, + startDate: startDate.length ? startDate : undefined, + endDate: endDate.length ? endDate : undefined, + }), + }); + }, + [history, location, toUrlParams, urlParams] + ); + + useEffect(() => { + setDataUsageMetricsFilters((prevState) => { + return { + ...prevState, + ...getDataUsageMetricsFiltersFromUrlParams(urlParams), + }; + }); + }, [setDataUsageMetricsFilters, urlParams]); + + return { + ...dataUsageMetricsFilters, + setUrlDataStreamsFilter, + setUrlDateRangeFilter, + setUrlMetricTypesFilter, + }; +}; diff --git a/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx new file mode 100644 index 0000000000000..b5407ae9e46d5 --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useState } from 'react'; +import type { + DurationRange, + OnRefreshChangeProps, +} from '@elastic/eui/src/components/date_picker/types'; +import { useDataUsageMetricsUrlParams } from './use_charts_url_params'; +import { DateRangePickerValues } from '../components/date_picker'; + +export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({ + autoRefreshOptions: { + enabled: false, + duration: 10000, + }, + startDate: 'now-24h/h', + endDate: 'now', + recentlyUsedDateRanges: [], +}); + +export const useDateRangePicker = () => { + const { + setUrlDateRangeFilter, + startDate: startDateFromUrl, + endDate: endDateFromUrl, + } = useDataUsageMetricsUrlParams(); + const [dateRangePickerState, setDateRangePickerState] = useState({ + ...DEFAULT_DATE_RANGE_OPTIONS, + startDate: startDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.startDate, + endDate: endDateFromUrl ?? DEFAULT_DATE_RANGE_OPTIONS.endDate, + }); + + const updateUsageMetricsDateRanges = useCallback( + ({ start, end }: DurationRange) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + startDate: start, + endDate: end, + })); + }, + [setDateRangePickerState] + ); + + const updateUsageMetricsRecentlyUsedDateRanges = useCallback( + (recentlyUsedDateRanges: DateRangePickerValues['recentlyUsedDateRanges']) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + recentlyUsedDateRanges, + })); + }, + [setDateRangePickerState] + ); + + // handle refresh timer update + const onRefreshChange = useCallback( + (evt: OnRefreshChangeProps) => { + setDateRangePickerState((prevState) => ({ + ...prevState, + autoRefreshOptions: { enabled: !evt.isPaused, duration: evt.refreshInterval }, + })); + }, + [setDateRangePickerState] + ); + + // handle manual time change on date picker + const onTimeChange = useCallback( + ({ start: newStart, end: newEnd }: DurationRange) => { + // update date ranges + updateUsageMetricsDateRanges({ start: newStart, end: newEnd }); + + // update recently used date ranges + const newRecentlyUsedDateRanges = [ + { start: newStart, end: newEnd }, + ...dateRangePickerState.recentlyUsedDateRanges + .filter( + (recentlyUsedRange: DurationRange) => + !(recentlyUsedRange.start === newStart && recentlyUsedRange.end === newEnd) + ) + .slice(0, 9), + ]; + updateUsageMetricsRecentlyUsedDateRanges(newRecentlyUsedDateRanges); + setUrlDateRangeFilter({ startDate: newStart, endDate: newEnd }); + }, + [ + dateRangePickerState.recentlyUsedDateRanges, + setUrlDateRangeFilter, + updateUsageMetricsDateRanges, + updateUsageMetricsRecentlyUsedDateRanges, + ] + ); + + return { dateRangePickerState, onRefreshChange, onTimeChange }; +}; diff --git a/x-pack/plugins/data_usage/public/app/types.ts b/x-pack/plugins/data_usage/public/app/types.ts new file mode 100644 index 0000000000000..13f53bc2ea6dd --- /dev/null +++ b/x-pack/plugins/data_usage/public/app/types.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MetricTypes } from '../../common/rest_types'; + +export type DataPoint = [number, number]; // [timestamp, value] + +export interface MetricSeries { + name: string; // Name of the data stream + data: DataPoint[]; // Array of data points in tuple format [timestamp, value] +} +// Use MetricTypes dynamically as keys for the Metrics interface +export type Metrics = Partial>; + +export interface MetricsResponse { + metrics: Metrics; +} +export interface MetricsResponse { + metrics: Metrics; +} diff --git a/x-pack/plugins/data_usage/public/application.tsx b/x-pack/plugins/data_usage/public/application.tsx index 1e6c35c4b8f0a..054aae397e5e1 100644 --- a/x-pack/plugins/data_usage/public/application.tsx +++ b/x-pack/plugins/data_usage/public/application.tsx @@ -16,6 +16,8 @@ import { PerformanceContextProvider } from '@kbn/ebt-tools'; import { useKibanaContextForPluginProvider } from './utils/use_kibana'; import { DataUsageStartDependencies, DataUsagePublicStart } from './types'; import { PLUGIN_ID } from '../common'; +import { DataUsage } from './app/data_usage'; +import { DataUsageReactQueryClientProvider } from '../common/query_client'; export const renderApp = ( core: CoreStart, @@ -51,7 +53,7 @@ const AppWithExecutionContext = ({ -
Data Usage
} /> +
@@ -76,7 +78,9 @@ const App = ({ core, plugins, pluginStart, params }: AppProps) => { return ( - + + + ); diff --git a/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts new file mode 100644 index 0000000000000..6b9860e997c12 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import type { IHttpFetchError } from '@kbn/core-http-browser'; +import { + UsageMetricsRequestSchemaQueryParams, + UsageMetricsResponseSchemaBody, +} from '../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../common'; +import { useKibanaContextForPlugin } from '../utils/use_kibana'; + +interface ErrorType { + statusCode: number; + message: string; +} + +export const useGetDataUsageMetrics = ( + query: UsageMetricsRequestSchemaQueryParams, + options: UseQueryOptions> = {} +): UseQueryResult> => { + const http = useKibanaContextForPlugin().services.http; + + return useQuery>({ + queryKey: ['get-data-usage-metrics', query], + ...options, + keepPreviousData: true, + queryFn: async () => { + return http.get(DATA_USAGE_METRICS_API_ROUTE, { + version: '1', + query: { + from: query.from, + to: query.to, + metricTypes: query.metricTypes, + dataStreams: query.dataStreams, + }, + }); + }, + }); +}; diff --git a/x-pack/plugins/data_usage/public/hooks/use_url_params.ts b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts new file mode 100644 index 0000000000000..865b71781df63 --- /dev/null +++ b/x-pack/plugins/data_usage/public/hooks/use_url_params.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { parse, stringify } from 'query-string'; +import { useLocation } from 'react-router-dom'; + +/** + * Parses `search` params and returns an object with them along with a `toUrlParams` function + * that allows being able to retrieve a stringified version of an object (default is the + * `urlParams` that was parsed) for use in the url. + * Object will be recreated every time `search` changes. + */ +export function useUrlParams>(): { + urlParams: T; + toUrlParams: (params?: T) => string; +} { + const { search } = useLocation(); + return useMemo(() => { + const urlParams = parse(search) as unknown as T; + return { + urlParams, + toUrlParams: (params: T = urlParams) => stringify(params as unknown as object), + }; + }, [search]); +} diff --git a/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx new file mode 100644 index 0000000000000..928ee73ad5280 --- /dev/null +++ b/x-pack/plugins/data_usage/public/utils/use_breadcrumbs.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChromeBreadcrumb, ChromeStart } from '@kbn/core-chrome-browser'; + +import { useEffect } from 'react'; +import { ManagementAppMountParams } from '@kbn/management-plugin/public'; + +export const useBreadcrumbs = ( + breadcrumbs: ChromeBreadcrumb[], + params: ManagementAppMountParams, + chromeService: ChromeStart +) => { + const { docTitle } = chromeService; + const isMultiple = breadcrumbs.length > 1; + + const docTitleValue = isMultiple ? breadcrumbs[breadcrumbs.length - 1].text : breadcrumbs[0].text; + + docTitle.change(docTitleValue as string); + + useEffect(() => { + params.setBreadcrumbs(breadcrumbs); + }, [breadcrumbs, params]); +}; diff --git a/x-pack/plugins/data_usage/server/common/errors.ts b/x-pack/plugins/data_usage/server/common/errors.ts new file mode 100644 index 0000000000000..7a43a10108be1 --- /dev/null +++ b/x-pack/plugins/data_usage/server/common/errors.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class BaseError extends Error { + constructor(message: string, public readonly meta?: MetaType) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/config.ts b/x-pack/plugins/data_usage/server/config.ts index 6453cce4f4d56..bf89431f2abea 100644 --- a/x-pack/plugins/data_usage/server/config.ts +++ b/x-pack/plugins/data_usage/server/config.ts @@ -6,9 +6,18 @@ */ import { schema, type TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from '@kbn/core/server'; -export const config = schema.object({ +export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: false }), }); -export type DataUsageConfig = TypeOf; +export type DataUsageConfigType = TypeOf; + +export const createConfig = (context: PluginInitializerContext): DataUsageConfigType => { + const pluginConfig = context.config.get>(); + + return { + ...pluginConfig, + }; +}; diff --git a/x-pack/plugins/data_usage/server/index.ts b/x-pack/plugins/data_usage/server/index.ts index 3aa49a184d003..66d839303d716 100644 --- a/x-pack/plugins/data_usage/server/index.ts +++ b/x-pack/plugins/data_usage/server/index.ts @@ -9,7 +9,7 @@ import type { PluginInitializerContext, PluginConfigDescriptor, } from '@kbn/core/server'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType } from './config'; import { DataUsagePlugin } from './plugin'; import type { @@ -19,11 +19,11 @@ import type { DataUsageStartDependencies, } from './types'; -import { config as configSchema } from './config'; +import { configSchema } from './config'; export type { DataUsageServerSetup, DataUsageServerStart }; -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, }; @@ -32,5 +32,5 @@ export const plugin: PluginInitializer< DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => +> = async (pluginInitializerContext: PluginInitializerContext) => await new DataUsagePlugin(pluginInitializerContext); diff --git a/x-pack/plugins/data_usage/server/plugin.ts b/x-pack/plugins/data_usage/server/plugin.ts index 8ab49d5104fff..2beae9b22bba9 100644 --- a/x-pack/plugins/data_usage/server/plugin.ts +++ b/x-pack/plugins/data_usage/server/plugin.ts @@ -7,13 +7,17 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; -import { DataUsageConfig } from './config'; +import { DataUsageConfigType, createConfig } from './config'; import type { + DataUsageContext, + DataUsageRequestHandlerContext, DataUsageServerSetup, DataUsageServerStart, DataUsageSetupDependencies, DataUsageStartDependencies, } from './types'; +import { registerDataUsageRoutes } from './routes'; +import { PLUGIN_ID } from '../common'; export class DataUsagePlugin implements @@ -24,11 +28,39 @@ export class DataUsagePlugin DataUsageStartDependencies > { - logger: Logger; - constructor(context: PluginInitializerContext) { + private readonly logger: Logger; + private dataUsageContext: DataUsageContext; + + constructor(context: PluginInitializerContext) { + const serverConfig = createConfig(context); + this.logger = context.logger.get(); + + this.logger.debug('data usage plugin initialized'); + this.dataUsageContext = { + logFactory: context.logger, + get serverConfig() { + return serverConfig; + }, + }; } setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup { + this.logger.debug('data usage plugin setup'); + pluginsSetup.features.registerElasticsearchFeature({ + id: PLUGIN_ID, + management: { + data: [PLUGIN_ID], + }, + privileges: [ + { + requiredClusterPrivileges: ['monitor'], + ui: [], + }, + ], + }); + const router = coreSetup.http.createRouter(); + registerDataUsageRoutes(router, this.dataUsageContext); + return {}; } @@ -36,5 +68,7 @@ export class DataUsagePlugin return {}; } - public stop() {} + public stop() { + this.logger.debug('Stopping data usage plugin'); + } } diff --git a/x-pack/plugins/data_usage/server/routes/error_handler.ts b/x-pack/plugins/data_usage/server/routes/error_handler.ts new file mode 100644 index 0000000000000..122df5e72b130 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/error_handler.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server'; +import { CustomHttpRequestError } from '../utils/custom_http_request_error'; +import { BaseError } from '../common/errors'; + +export class NotFoundError extends BaseError {} + +/** + * Default Data Usage Routes error handler + * @param logger + * @param res + * @param error + */ +export const errorHandler = ( + logger: Logger, + res: KibanaResponseFactory, + error: E +): IKibanaResponse => { + logger.error(error); + + if (error instanceof CustomHttpRequestError) { + return res.customError({ + statusCode: error.statusCode, + body: error, + }); + } + + if (error instanceof NotFoundError) { + return res.notFound({ body: error }); + } + + // Kibana CORE will take care of `500` errors when the handler `throw`'s, including logging the error + throw error; +}; diff --git a/x-pack/plugins/data_usage/server/routes/index.tsx b/x-pack/plugins/data_usage/server/routes/index.tsx new file mode 100644 index 0000000000000..b6b80c38864f3 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataUsageContext, DataUsageRouter } from '../types'; +import { registerDataStreamsRoute, registerUsageMetricsRoute } from './internal'; + +export const registerDataUsageRoutes = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + registerUsageMetricsRoute(router, dataUsageContext); + registerDataStreamsRoute(router, dataUsageContext); +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts new file mode 100644 index 0000000000000..0d71d93b55849 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStreamsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; + +import { getDataStreamsHandler } from './data_streams_handler'; + +export const registerDataStreamsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: DATA_USAGE_DATA_STREAMS_API_ROUTE, + }) + .addVersion( + { + version: '1', + validate: { + request: {}, + response: { + 200: DataStreamsResponseSchema, + }, + }, + }, + getDataStreamsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts new file mode 100644 index 0000000000000..686edd0c4f4b7 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; + +import { errorHandler } from '../error_handler'; + +export const getDataStreamsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler => { + const logger = dataUsageContext.logFactory.get('dataStreamsRoute'); + + return async (context, _, response) => { + logger.debug(`Retrieving user data streams`); + + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + const { data_streams: dataStreamsResponse } = await esClient.indices.dataStreamsStats({ + name: '*', + expand_wildcards: 'all', + }); + + const sorted = dataStreamsResponse + .sort((a, b) => b.store_size_bytes - a.store_size_bytes) + .map((dataStream) => ({ + name: dataStream.data_stream, + storageSizeBytes: dataStream.store_size_bytes, + })); + return response.ok({ + body: sorted, + }); + } catch (error) { + return errorHandler(logger, response, error); + } + }; +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/index.tsx b/x-pack/plugins/data_usage/server/routes/internal/index.tsx new file mode 100644 index 0000000000000..e8d874bb7e6af --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/index.tsx @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { registerUsageMetricsRoute } from './usage_metrics'; +export { registerDataStreamsRoute } from './data_streams'; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts new file mode 100644 index 0000000000000..5bf3008ef668a --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UsageMetricsRequestSchema, UsageMetricsResponseSchema } from '../../../common/rest_types'; +import { DATA_USAGE_METRICS_API_ROUTE } from '../../../common'; +import { DataUsageContext, DataUsageRouter } from '../../types'; + +import { getUsageMetricsHandler } from './usage_metrics_handler'; + +export const registerUsageMetricsRoute = ( + router: DataUsageRouter, + dataUsageContext: DataUsageContext +) => { + if (dataUsageContext.serverConfig.enabled) { + router.versioned + .get({ + access: 'internal', + path: DATA_USAGE_METRICS_API_ROUTE, + }) + .addVersion( + { + version: '1', + validate: { + request: UsageMetricsRequestSchema, + response: { + 200: UsageMetricsResponseSchema, + }, + }, + }, + getUsageMetricsHandler(dataUsageContext) + ); + } +}; diff --git a/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts new file mode 100644 index 0000000000000..6f992c9fb2a38 --- /dev/null +++ b/x-pack/plugins/data_usage/server/routes/internal/usage_metrics_handler.ts @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandler } from '@kbn/core/server'; +import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types'; +import { + MetricTypes, + UsageMetricsRequestSchemaQueryParams, + UsageMetricsResponseSchema, +} from '../../../common/rest_types'; +import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types'; + +import { errorHandler } from '../error_handler'; + +const formatStringParams = (value: T | T[]): T[] | MetricTypes[] => + typeof value === 'string' ? [value] : value; + +export const getUsageMetricsHandler = ( + dataUsageContext: DataUsageContext +): RequestHandler< + never, + UsageMetricsRequestSchemaQueryParams, + unknown, + DataUsageRequestHandlerContext +> => { + const logger = dataUsageContext.logFactory.get('usageMetricsRoute'); + + return async (context, request, response) => { + try { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + + // @ts-ignore + const { from, to, metricTypes, dataStreams: dsNames, size } = request.query; + logger.debug(`Retrieving usage metrics`); + + const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse = + await esClient.indices.getDataStream({ + name: '*', + expand_wildcards: 'all', + }); + + const hasDataStreams = dataStreamsResponse.length > 0; + let userDsNames: string[] = []; + + if (dsNames?.length) { + userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames; + } else if (!userDsNames.length && hasDataStreams) { + userDsNames = dataStreamsResponse.map((ds) => ds.name); + } + + // If no data streams are found, return an empty response + if (!userDsNames.length) { + return response.ok({ + body: { + metrics: {}, + }, + }); + } + + const metrics = await fetchMetricsFromAutoOps({ + from, + to, + metricTypes: formatStringParams(metricTypes) as MetricTypes[], + dataStreams: formatStringParams(userDsNames), + }); + + return response.ok({ + body: { + metrics, + }, + }); + } catch (error) { + logger.error(`Error retrieving usage metrics: ${error.message}`); + return errorHandler(logger, response, error); + } + }; +}; + +const fetchMetricsFromAutoOps = async ({ + from, + to, + metricTypes, + dataStreams, +}: { + from: string; + to: string; + metricTypes: MetricTypes[]; + dataStreams: string[]; +}) => { + // TODO: fetch data from autoOps using userDsNames + /* + const response = await axios.post('https://api.auto-ops.{region}.{csp}.cloud.elastic.co/monitoring/serverless/v1/projects/{project_id}/metrics', { + from: Date.parse(from), + to: Date.parse(to), + metric_types: metricTypes, + allowed_indices: dataStreams, + }); + const { data } = response;*/ + // mock data from autoOps https://github.com/elastic/autoops-services/blob/master/monitoring/service/specs/serverless_project_metrics_api.yaml + const mockData = { + metrics: { + ingest_rate: [ + { + name: 'metrics-apache_spark.driver-default', + data: [ + [1726858530000, 13756849], + [1726862130000, 14657904], + [1726865730000, 12798561], + [1726869330000, 13578213], + [1726872930000, 14123495], + [1726876530000, 13876548], + [1726880130000, 12894561], + [1726883730000, 14478953], + [1726887330000, 14678905], + [1726890930000, 13976547], + [1726894530000, 14568945], + [1726898130000, 13789561], + [1726901730000, 14478905], + [1726905330000, 13956423], + [1726908930000, 14598234], + ], + }, + { + name: 'logs-apm.app.adservice-default', + data: [ + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], + ], + }, + { + name: 'metrics-apm.app.aws-lambdas-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + ], + storage_retained: [ + { + name: 'metrics-apache_spark.driver-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + { + name: 'logs-apm.app.adservice-default', + data: [ + [1726858530000, 12894623], + [1726862130000, 14436905], + [1726865730000, 13794805], + [1726869330000, 14048532], + [1726872930000, 14237495], + [1726876530000, 13745689], + [1726880130000, 13974562], + [1726883730000, 14234653], + [1726887330000, 14323479], + [1726890930000, 14023945], + [1726894530000, 14189673], + [1726898130000, 14247895], + [1726901730000, 14098324], + [1726905330000, 14478905], + [1726908930000, 14323894], + ], + }, + { + name: 'metrics-apm.app.aws-lambdas-default', + data: [ + [1726858530000, 12576413], + [1726862130000, 13956423], + [1726865730000, 14568945], + [1726869330000, 14234856], + [1726872930000, 14368942], + [1726876530000, 13897654], + [1726880130000, 14456989], + [1726883730000, 14568956], + [1726887330000, 13987562], + [1726890930000, 14567894], + [1726894530000, 14246789], + [1726898130000, 14567895], + [1726901730000, 14457896], + [1726905330000, 14567895], + [1726908930000, 13989456], + ], + }, + ], + }, + }; + // Make sure data is what we expect + const validatedData = UsageMetricsResponseSchema.body().validate(mockData); + + return validatedData.metrics; +}; diff --git a/x-pack/plugins/data_usage/server/types/index.ts b/x-pack/plugins/data_usage/server/types/index.ts new file mode 100644 index 0000000000000..6cc0ccaa93a6d --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; diff --git a/x-pack/plugins/data_usage/server/types/types.ts b/x-pack/plugins/data_usage/server/types/types.ts new file mode 100644 index 0000000000000..c90beb184f020 --- /dev/null +++ b/x-pack/plugins/data_usage/server/types/types.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CoreRequestHandlerContext, + CustomRequestHandlerContext, + IRouter, + LoggerFactory, +} from '@kbn/core/server'; +import { DeepReadonly } from 'utility-types'; +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { DataUsageConfigType } from '../config'; + +export interface DataUsageSetupDependencies { + features: FeaturesPluginSetup; +} + +/* eslint-disable @typescript-eslint/no-empty-interface*/ +export interface DataUsageStartDependencies {} + +export interface DataUsageServerSetup {} + +export interface DataUsageServerStart {} + +interface DataUsageApiRequestHandlerContext { + core: CoreRequestHandlerContext; +} + +export type DataUsageRequestHandlerContext = CustomRequestHandlerContext<{ + dataUsage: DataUsageApiRequestHandlerContext; +}>; + +export type DataUsageRouter = IRouter; + +export interface DataUsageContext { + logFactory: LoggerFactory; + serverConfig: DeepReadonly; +} diff --git a/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts new file mode 100644 index 0000000000000..a7f00a0e82a3d --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/custom_http_request_error.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class CustomHttpRequestError extends Error { + constructor( + message: string, + public readonly statusCode: number = 500, + public readonly meta?: unknown + ) { + super(message); + // For debugging - capture name of subclasses + this.name = this.constructor.name; + + if (meta instanceof Error) { + this.stack += `\n----- original error -----\n${meta.stack}`; + } + } +} diff --git a/x-pack/plugins/data_usage/server/utils/index.ts b/x-pack/plugins/data_usage/server/utils/index.ts new file mode 100644 index 0000000000000..af46a18f61a79 --- /dev/null +++ b/x-pack/plugins/data_usage/server/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { CustomHttpRequestError } from './custom_http_request_error'; diff --git a/x-pack/plugins/data_usage/tsconfig.json b/x-pack/plugins/data_usage/tsconfig.json index ebc023568cf88..cecbeb654db30 100644 --- a/x-pack/plugins/data_usage/tsconfig.json +++ b/x-pack/plugins/data_usage/tsconfig.json @@ -13,6 +13,7 @@ "kbn_references": [ "@kbn/core", "@kbn/i18n", + "@kbn/data-plugin", "@kbn/kibana-react-plugin", "@kbn/management-plugin", "@kbn/react-kibana-context-render", @@ -21,6 +22,12 @@ "@kbn/share-plugin", "@kbn/config-schema", "@kbn/logging", + "@kbn/deeplinks-observability", + "@kbn/unified-search-plugin", + "@kbn/i18n-react", + "@kbn/core-http-browser", + "@kbn/core-chrome-browser", + "@kbn/features-plugin", ], "exclude": ["target/**/*"] }