diff --git a/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts b/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts index ecacf24059a09..bd3937c3677ce 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts @@ -260,6 +260,8 @@ export const dataStreamDetailsRt = rt.partial({ services: rt.record(rt.string, rt.array(rt.string)), hosts: rt.record(rt.string, rt.array(rt.string)), userPrivileges: userPrivilegesRt, + defaultRetentionPeriod: rt.string, + customRetentionPeriod: rt.string, }); export type DataStreamDetails = rt.TypeOf; @@ -308,3 +310,9 @@ export const getPreviewChartResponseRt = rt.type({ }); export type PreviewChartResponse = rt.TypeOf; + +export const updateFailureStoreResponseRt = rt.type({ + headers: rt.record(rt.string, rt.unknown), +}); + +export type UpdateFailureStoreResponse = rt.TypeOf; diff --git a/x-pack/platform/plugins/shared/dataset_quality/common/data_stream_details/types.ts b/x-pack/platform/plugins/shared/dataset_quality/common/data_stream_details/types.ts index f150e111847cd..e30c79e7b242d 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/common/data_stream_details/types.ts @@ -27,3 +27,9 @@ export interface IntegrationType { areAssetsAvailable: boolean; integration?: Integration; } + +export interface UpdateFailureStoreParams { + dataStream: string; + failureStoreEnabled: boolean; + customRetentionPeriod?: string; +} diff --git a/x-pack/platform/plugins/shared/dataset_quality/common/translations.ts b/x-pack/platform/plugins/shared/dataset_quality/common/translations.ts index 50f0391541a73..a13b6f9fdfbe6 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/common/translations.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/common/translations.ts @@ -331,6 +331,13 @@ export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate( } ); +export const overviewPanelDatasetQualityIndicatorNoFailureStore = i18n.translate( + 'xpack.datasetQuality.details.overviewPanel.datasetQuality.noFailureStore', + { + defaultMessage: 'No failure store', + } +); + export const overviewDegradedFieldsTableLoadingText = i18n.translate( 'xpack.datasetQuality.details.degradedFieldsTableLoadingText', { diff --git a/x-pack/platform/plugins/shared/dataset_quality/kibana.jsonc b/x-pack/platform/plugins/shared/dataset_quality/kibana.jsonc index 800b0a3ef5a15..858b376963ef4 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/kibana.jsonc +++ b/x-pack/platform/plugins/shared/dataset_quality/kibana.jsonc @@ -39,6 +39,7 @@ "requiredBundles": [ "charts", "discover", + "esUiShared", "stackAlerts", ], "extraPublicDirs": [ diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/card.tsx b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/card.tsx index 18f26da7da9c0..7040ff98f81fc 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/card.tsx +++ b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/card.tsx @@ -26,32 +26,46 @@ export function Card({ }) { const { euiTheme } = useEuiTheme(); - return ( + const style = css` + height: 100%; + min-width: 300px; + border: ${isSelected + ? `${euiTheme.border.width.thick} solid ${euiTheme.colors.borderStrongPrimary}` + : 'none'}; + background-color: ${isSelected ? euiTheme.colors.backgroundLightPrimary : 'inherit'}; + `; + + const dataTestSubject = `datasetQualityDetailsSummaryKpiCard-${title}`; + + const content = ( + <> + {title} + + +

{kpiValue}

+
+ + {footer} + + ); + + return onClick ? ( - {title} - - -

{kpiValue}

-
- - {footer} + {content}
+ ) : ( +
+ {content} +
); } diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/index.tsx b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/index.tsx index 03aeed82e564c..2785e963614aa 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/index.tsx +++ b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/quality_summary_cards/index.tsx @@ -5,18 +5,19 @@ * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui'; -import React from 'react'; +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; +import { FailureStoreModal } from '@kbn/failure-store-modal'; import { overviewPanelDatasetQualityIndicatorDegradedDocs, overviewPanelDatasetQualityIndicatorFailedDocs, + overviewPanelDatasetQualityIndicatorNoFailureStore, } from '../../../../../common/translations'; import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; import { useQualityIssuesDocsChart } from '../../../../hooks/use_quality_issues_docs_chart'; import { useDatasetQualityDetailsState } from '../../../../hooks/use_dataset_quality_details_state'; import { DatasetQualityIndicator, QualityPercentageIndicator } from '../../../quality_indicator'; -import { useKibanaContextForPlugin } from '../../../../utils/use_kibana'; import { Card } from './card'; // Allow for lazy loading @@ -40,20 +41,34 @@ export default function QualitySummaryCards({ } = useOverviewSummaryPanel(); const { handleDocsTrendChartChange } = useQualityIssuesDocsChart(); const { - dataStream, canUserReadFailureStore, hasFailureStore, loadingState: { dataStreamSettingsLoading }, + defaultRetentionPeriod, + customRetentionPeriod, + updateFailureStore, } = useDatasetQualityDetailsState(); - const { - services: { - share: { url: urlService }, - }, - } = useKibanaContextForPlugin(); + const [isFailureStoreModalOpen, setIsFailureStoreModalOpen] = useState(false); + + const closeModal = () => { + setIsFailureStoreModalOpen(false); + }; - const locator = urlService.locators.get('INDEX_MANAGEMENT_LOCATOR_ID'); - const locatorParams = { page: 'data_streams_details', dataStreamName: dataStream } as const; + const handleSaveModal = async (data: { + failureStoreEnabled: boolean; + customRetentionPeriod?: string; + }) => { + updateFailureStore({ + failureStoreEnabled: data.failureStoreEnabled, + customRetentionPeriod: data.customRetentionPeriod, + }); + closeModal(); + }; + + const onClick = () => { + setIsFailureStoreModalOpen(true); + }; return ( @@ -95,26 +110,38 @@ export default function QualitySummaryCards({ {!dataStreamSettingsLoading && !(hasFailureStore && canUserReadFailureStore) ? ( - - {i18n.translate('xpack.datasetQuality.enableFailureStore', { - defaultMessage: 'Enable failure store', - })} - - ) - } - /> + <> + + {i18n.translate('xpack.datasetQuality.enableFailureStore', { + defaultMessage: 'Enable failure store', + })} + + ) + } + /> + {canUserReadFailureStore && isFailureStoreModalOpen && defaultRetentionPeriod && ( + + )} + ) : ( { [service] ); + const updateFailureStore = useCallback( + ({ + failureStoreEnabled, + customRetentionPeriod, + }: { + failureStoreEnabled: boolean; + customRetentionPeriod?: string; + }) => { + service.send({ + type: 'UPDATE_FAILURE_STORE', + data: { + ...dataStreamDetails, + hasFailureStore: failureStoreEnabled, + customRetentionPeriod, + }, + }); + }, + [dataStreamDetails, service] + ); + const hasFailureStore = Boolean(dataStreamDetails?.hasFailureStore); const canShowFailureStoreInfo = canUserReadFailureStore && hasFailureStore; + const defaultRetentionPeriod = dataStreamDetails?.defaultRetentionPeriod; + const customRetentionPeriod = dataStreamDetails?.customRetentionPeriod; return { service, @@ -180,6 +202,7 @@ export const useDatasetQualityDetailsState = () => { timeRange, loadingState, updateTimeRange, + updateFailureStore, dataStreamSettings, integrationDetails, canUserAccessDashboards, @@ -189,5 +212,7 @@ export const useDatasetQualityDetailsState = () => { canShowFailureStoreInfo, expandedQualityIssue, isQualityIssueFlyoutOpen, + defaultRetentionPeriod, + customRetentionPeriod, }; }; diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 5955d8010e48a..d89ca3c7f4fd4 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -16,6 +16,7 @@ import type { FailedDocsErrorsResponse, IntegrationDashboardsResponse, UpdateFieldLimitResponse, + UpdateFailureStoreResponse, } from '../../../common/api_types'; import { checkAndLoadIntegrationResponseRt, @@ -29,6 +30,7 @@ import { integrationDashboardsRT, qualityIssueBaseRT, updateFieldLimitResponseRt, + updateFailureStoreResponseRt, } from '../../../common/api_types'; import type { DataStreamDetails, @@ -309,4 +311,34 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { new DatasetQualityError(`Failed to decode rollover response: ${message}"`) )(response); } + + public async updateFailureStore({ + dataStream, + failureStoreEnabled, + customRetentionPeriod, + }: { + dataStream: string; + failureStoreEnabled: boolean; + customRetentionPeriod?: string; + }): Promise { + const response = await this.http + .put( + `/internal/dataset_quality/data_streams/${dataStream}/update_failure_store`, + { + body: JSON.stringify({ + failureStoreEnabled, + customRetentionPeriod, + }), + } + ) + .catch((error) => { + throw new DatasetQualityError(`Failed to update failure store": ${error}`, error); + }); + + return decodeOrThrow( + updateFailureStoreResponseRt, + (message: string) => + new DatasetQualityError(`Failed to decode update failure store response: ${message}"`) + )(response); + } } diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/types.ts index db9948e94210d..a5aba8c66204c 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/services/data_stream_details/types.ts @@ -23,6 +23,7 @@ import type { IntegrationType, CheckAndLoadIntegrationParams, UpdateFieldLimitParams, + UpdateFailureStoreParams, } from '../../../common/data_stream_details/types'; import type { Dashboard, @@ -32,6 +33,7 @@ import type { FailedDocsDetails, FailedDocsErrorsResponse, UpdateFieldLimitResponse, + UpdateFailureStoreResponse, } from '../../../common/api_types'; export type DataStreamDetailsServiceSetup = void; @@ -62,4 +64,5 @@ export interface IDataStreamDetailsClient { analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; setNewFieldLimit(params: UpdateFieldLimitParams): Promise; rolloverDataStream(params: { dataStream: string }): Promise; + updateFailureStore(params: UpdateFailureStoreParams): Promise; } diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts index fbd2dd1dc1913..10eb6d660aaa5 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts @@ -77,3 +77,20 @@ export const rolloverDataStreamFailedNotifier = ( text: error.message, }); }; + +export const updateFailureStoreFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.details.updateFailureStoreFailed', { + defaultMessage: "We couldn't update the failure store settings.", + }), + text: error.message, + }); +}; + +export const updateFailureStoreSuccessNotifier = (toasts: IToasts) => { + toasts.addSuccess({ + title: i18n.translate('xpack.datasetQuality.details.updateFailureStoreSuccess', { + defaultMessage: 'Failure store settings saved', + }), + }); +}; diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 2e34ae8b787b5..b88d7613fa911 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -41,6 +41,8 @@ import { fetchIntegrationDashboardsFailedNotifier, rolloverDataStreamFailedNotifier, updateFieldLimitFailedNotifier, + updateFailureStoreFailedNotifier, + updateFailureStoreSuccessNotifier, } from './notifications'; import { filterIssues, @@ -531,6 +533,32 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( ], }, }, + failureStoreUpdate: { + initial: 'idle', + states: { + idle: { + on: { + UPDATE_FAILURE_STORE: { + target: 'updating', + actions: ['storeDataStreamDetails'], + }, + }, + }, + updating: { + invoke: { + src: 'updateFailureStore', + onDone: { + target: 'idle', + actions: ['notifyUpdateFailureStoreSuccess', 'raiseForceTimeRangeRefresh'], + }, + onError: { + target: 'idle', + actions: ['notifyUpdateFailureStoreFailed', 'raiseForceTimeRangeRefresh'], + }, + }, + }, + }, + }, }, }, indexNotFound: { @@ -818,6 +846,9 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ updateFieldLimitFailedNotifier(toasts, event.data), notifyRolloverDataStreamError: (context, event: DoneInvokeEvent) => rolloverDataStreamFailedNotifier(toasts, event.data, context.dataStream), + notifyUpdateFailureStoreSuccess: () => updateFailureStoreSuccessNotifier(toasts), + notifyUpdateFailureStoreFailed: (_context, event: DoneInvokeEvent) => + updateFailureStoreFailedNotifier(toasts, event.data), }, services: { checkDatasetIsAggregatable: (context) => { @@ -961,6 +992,21 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ dataStream: context.dataStream, }); }, + updateFailureStore: (context) => { + if ( + 'dataStreamDetails' in context && + context.dataStreamDetails && + context.dataStreamDetails.hasFailureStore + ) { + return dataStreamDetailsClient.updateFailureStore({ + dataStream: context.dataStream, + failureStoreEnabled: context.dataStreamDetails.hasFailureStore, + customRetentionPeriod: context.dataStreamDetails.customRetentionPeriod, + }); + } + + return Promise.resolve(); + }, }, }); diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index 1ed0d0f6fc934..2eef587809371 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -20,6 +20,7 @@ import type { NonAggregatableDatasets, QualityIssue, UpdateFieldLimitResponse, + UpdateFailureStoreResponse, } from '../../../common/api_types'; import type { IntegrationType } from '../../../common/data_stream_details'; import type { TableCriteria, TimeRangeConfig } from '../../../common/types'; @@ -229,6 +230,14 @@ export type DatasetQualityDetailsControllerTypeState = WithDegradeFieldAnalysis & WithNewFieldLimit & WithNewFieldLimitResponse; + } + | { + value: 'initializing.failureStoreUpdate.idle'; + context: WithDefaultControllerState; + } + | { + value: 'initializing.failureStoreUpdate.updating'; + context: WithDefaultControllerState; }; export type DatasetQualityDetailsControllerContext = @@ -275,6 +284,10 @@ export type DatasetQualityDetailsControllerEvent = | { type: 'ROLLOVER_DATA_STREAM'; } + | { + type: 'UPDATE_FAILURE_STORE'; + dataStreamsDetails: DataStreamDetails; + } | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent @@ -288,4 +301,5 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent; diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 71f6306df3e5a..c94708a173aff 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -92,6 +92,7 @@ export async function getDataStreamDetails({ canMonitor: dataStreamPrivileges.monitor, canReadFailureStore: dataStreamPrivileges[FAILURE_STORE_PRIVILEGE], }, + customRetentionPeriod: esDataStream?.customRetentionPeriod, }; } catch (e) { // Respond with empty object if data stream does not exist diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index 14d5fcd8c3365..fdba06727a2c4 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -73,6 +73,8 @@ export async function getDataStreams(options: { canReadFailureStore: dataStreamsPrivileges[dataStream.name][FAILURE_STORE_PRIVILEGE], }, hasFailureStore: dataStream.failure_store?.enabled, + // @ts-expect-error + customRetentionPeriod: dataStream.failure_store?.lifecycle?.data_retention, })); return { diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams_default_retention_period/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams_default_retention_period/index.ts new file mode 100644 index 0000000000000..ae290549444a5 --- /dev/null +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams_default_retention_period/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; + +// In case this retention is not present in cluster. +// This is extracted from the docs that indicate that a thirty day (30d) retention is applied to failure store data: +// https://www.elastic.co/docs/manage-data/data-store/data-streams/failure-store#manage-failure-store-lifecycle +const DEFAULT_RETENTION_PERIOD = '30d'; + +export async function getDataStreamDefaultRetentionPeriod({ + esClient, +}: { + esClient: ElasticsearchClient; +}) { + const { persistent, defaults } = await esClient.cluster.getSettings({ include_defaults: true }); + const persistentDSRetention = persistent?.data_streams?.lifecycle?.retention?.failures_default; + const defaultsDSRetention = defaults?.data_streams?.lifecycle?.retention?.failures_default; + return persistentDSRetention ?? defaultsDSRetention ?? DEFAULT_RETENTION_PERIOD; +} diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/routes.ts index 3b74a7fd5f10f..13c24ac883b29 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/routes.ts @@ -20,6 +20,7 @@ import type { DegradedFieldResponse, DegradedFieldValues, NonAggregatableDatasets, + UpdateFailureStoreResponse, UpdateFieldLimitResponse, } from '../../../common/api_types'; import { datasetQualityPrivileges } from '../../services'; @@ -41,6 +42,8 @@ import { getDegradedFields } from './get_degraded_fields'; import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; import { updateFieldLimit } from './update_field_limit'; import { getDataStreamsCreationDate } from './get_data_streams_creation_date'; +import { updateFailureStore } from './update_failure_store'; +import { getDataStreamDefaultRetentionPeriod } from './get_data_streams_default_retention_period'; const datasetTypesPrivilegesRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/types_privileges', @@ -476,6 +479,7 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({ const esClient = coreContext.elasticsearch.client; const isServerless = (await getEsCapabilities()).serverless; + const dataStreamDetails = await getDataStreamDetails({ esClient, dataStream, @@ -484,7 +488,16 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({ isServerless, }); - return dataStreamDetails; + // If dataStreamDetails is empty, return empty object, otherwise append defaultRetentionPeriod + if (!dataStreamDetails || Object.keys(dataStreamDetails).length === 0) { + return {} as DataStreamDetails; + } + + const defaultRetentionPeriod = await getDataStreamDefaultRetentionPeriod({ + esClient: esClient.asSecondaryAuthUser, + }); + + return { ...dataStreamDetails, defaultRetentionPeriod }; }, }); @@ -592,6 +605,45 @@ const rolloverDataStream = createDatasetQualityServerRoute({ }, }); +const updateFailureStoreRoute = createDatasetQualityServerRoute({ + endpoint: 'PUT /internal/dataset_quality/data_streams/{dataStream}/update_failure_store', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + body: t.type({ + failureStoreEnabled: t.boolean, + customRetentionPeriod: t.union([t.string, t.undefined]), + }), + }), + options: { + tags: [], + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + async handler(resources): Promise { + const { context, params, getEsCapabilities } = resources; + const coreContext = await context.core; + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const isServerless = (await getEsCapabilities()).serverless; + + const updatedLimitResponse = await updateFailureStore({ + esClient, + dataStream: params.path.dataStream, + failureStoreEnabled: params.body.failureStoreEnabled, + customRetentionPeriod: params.body.customRetentionPeriod, + isServerless, + }); + + return updatedLimitResponse; + }, +}); + export const dataStreamsRouteRepository = { ...datasetTypesPrivilegesRoute, ...statsRoute, @@ -608,4 +660,5 @@ export const dataStreamsRouteRepository = { ...updateFieldLimitRoute, ...rolloverDataStream, ...failedDocsRouteRepository, + ...updateFailureStoreRoute, }; diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/update_failure_store/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/update_failure_store/index.ts new file mode 100644 index 0000000000000..dc8a8a978689e --- /dev/null +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/update_failure_store/index.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { badRequest } from '@hapi/boom'; +import type { UpdateFailureStoreResponse } from '../../../../common/api_types'; + +export async function updateFailureStore({ + esClient, + dataStream, + failureStoreEnabled, + customRetentionPeriod, + isServerless, +}: { + esClient: ElasticsearchClient; + dataStream: string; + failureStoreEnabled: boolean; + customRetentionPeriod?: string; + isServerless: boolean; +}): Promise { + try { + return await esClient.indices.putDataStreamOptions( + { + name: dataStream, + failure_store: { + enabled: failureStoreEnabled, + lifecycle: { + data_retention: customRetentionPeriod, + ...(isServerless ? {} : { enabled: !!customRetentionPeriod }), + }, + }, + }, + { meta: true } + ); + } catch (error) { + throw badRequest(`Failed to update failure store for data stream "${dataStream}": ${error}`); + } +} diff --git a/x-pack/platform/plugins/shared/dataset_quality/tsconfig.json b/x-pack/platform/plugins/shared/dataset_quality/tsconfig.json index ba75226cc6cd9..28545cd4d9a75 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/tsconfig.json +++ b/x-pack/platform/plugins/shared/dataset_quality/tsconfig.json @@ -70,6 +70,7 @@ "@kbn/unified-search-plugin", "@kbn/usage-collection-plugin", "@kbn/xstate-utils", + "@kbn/failure-store-modal" ], "exclude": [ "target/**/*" diff --git a/x-pack/solutions/observability/test/functional/apps/dataset_quality/dataset_quality_details_failure_store.ts b/x-pack/solutions/observability/test/functional/apps/dataset_quality/dataset_quality_details_failure_store.ts new file mode 100644 index 0000000000000..8fda4475addf7 --- /dev/null +++ b/x-pack/solutions/observability/test/functional/apps/dataset_quality/dataset_quality_details_failure_store.ts @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { IndexTemplateName } from '@kbn/apm-synthtrace/src/lib/logs/custom_logsdb_index_templates'; +import type { DatasetQualityFtrProviderContext } from './config'; +import { + createFailedLogRecord, + datasetNames, + defaultNamespace, + getInitialTestLogs, + getLogsForDataset, + processors, +} from './data'; +import { + createDatasetQualityUserWithRole, + deleteDatasetQualityUserWithRole, +} from './roles/role_management'; + +export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) { + const PageObjects = getPageObjects(['datasetQuality', 'security']); + const testSubjects = getService('testSubjects'); + const synthtrace = getService('logSynthtraceEsClient'); + const security = getService('security'); + const retry = getService('retry'); + const to = '2024-01-01T12:00:00.000Z'; + + const failureStoreDatasetName = datasetNames[0]; + const failureStoreDataStreamName = `logs-${failureStoreDatasetName}-${defaultNamespace}`; + const nofailureStoreDatasetName = datasetNames[1]; + const nofailureStoreDataStreamName = `logs-${nofailureStoreDatasetName}-${defaultNamespace}`; + + describe('Dataset quality details failure store', function () { + // This disables the forward-compatibility test for Elasticsearch 8.19 with Kibana and ES 9.0. + // These versions are not expected to work together. Note: Failure store is not available in ES 9.0, + // and running these tests will result in an "unknown index privilege [read_failure_store]" error. + this.onlyEsVersion('8.19 || >=9.1'); + + before(async () => { + await synthtrace.createCustomPipeline(processors, 'synth.2@pipeline'); + await synthtrace.createComponentTemplate({ + name: 'synth.2@custom', + dataStreamOptions: { + failure_store: { + enabled: false, + }, + }, + }); + await synthtrace.createIndexTemplate(IndexTemplateName.Synht2); + + await synthtrace.index([ + // Ingest basic logs + getInitialTestLogs({ to, count: 4 }), + createFailedLogRecord({ + to: new Date().toISOString(), + count: 2, + dataset: failureStoreDataStreamName, + }), + getLogsForDataset({ + to: new Date().toISOString(), + count: 4, + dataset: failureStoreDataStreamName, + }), + getLogsForDataset({ + to: new Date().toISOString(), + count: 4, + dataset: nofailureStoreDatasetName, + }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await synthtrace.deleteIndexTemplate(IndexTemplateName.Synht2); + await synthtrace.deleteComponentTemplate('synth.2@custom'); + await synthtrace.deleteCustomPipeline('synth.2@pipeline'); + }); + + describe('without failure store permissions', () => { + before(async () => { + await createDatasetQualityUserWithRole(security, 'canNotReadFailureStore'); + + await PageObjects.security.forceLogout(); + await PageObjects.security.login( + 'canNotReadFailureStore', + 'canNotReadFailureStore-password', + { + expectSpaceSelector: false, + } + ); + }); + + after(async () => { + await deleteDatasetQualityUserWithRole(security, 'canNotReadFailureStore'); + }); + + it('should show "No failure store" card when failure store is disabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nofailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + + it('should show "No failure store" card when failure store is enabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: failureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + }); + + describe('with failure store permissions', () => { + before(async () => { + await createDatasetQualityUserWithRole(security, 'fullAccess'); + + await PageObjects.security.forceLogout(); + await PageObjects.security.login('fullAccess', 'fullAccess-password', { + expectSpaceSelector: false, + }); + }); + + after(async () => { + await deleteDatasetQualityUserWithRole(security, 'fullAccess'); + }); + + it('should show "No failure store" card when failure store is disabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nofailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + + it('should open failure store modal and save new config', async () => { + const { + datasetQualityDetailsSummaryCardFailedDocuments, + datasetQualityDetailsSummaryCardNoFailureStore, + datasetQualityDetailsEnableFailureStoreButton, + editFailureStoreModal, + failureStoreModalSaveButton, + enableFailureStoreToggle, + } = PageObjects.datasetQuality.testSubjectSelectors; + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: nofailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + await testSubjects.missingOrFail(datasetQualityDetailsSummaryCardFailedDocuments); + + await testSubjects.click(datasetQualityDetailsEnableFailureStoreButton); + + await testSubjects.existOrFail(editFailureStoreModal); + + const saveModalButton = await testSubjects.find(failureStoreModalSaveButton); + expect(await saveModalButton.isEnabled()).to.be(false); + + await testSubjects.click(enableFailureStoreToggle); + + await retry.try(async () => { + expect(await saveModalButton.isEnabled()).to.be(true); + }); + + await testSubjects.click(failureStoreModalSaveButton); + + await testSubjects.missingOrFail(editFailureStoreModal); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardFailedDocuments); + + await testSubjects.missingOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardFailedDocuments + ); + expect(failedDocsCard).to.contain('Failed documents'); + }); + + it('should show failed docs count when failure store is enabled', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: failureStoreDataStreamName, + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsSummaryCardFailedDocuments + ); + const failedDocsCard = await testSubjects.getVisibleText( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsSummaryCardFailedDocuments + ); + expect(failedDocsCard).to.not.contain('No failure store'); + }); + }); + }); +} diff --git a/x-pack/solutions/observability/test/functional/apps/dataset_quality/index.ts b/x-pack/solutions/observability/test/functional/apps/dataset_quality/index.ts index 455bb3b20c768..84ff1ac25bfa6 100644 --- a/x-pack/solutions/observability/test/functional/apps/dataset_quality/index.ts +++ b/x-pack/solutions/observability/test/functional/apps/dataset_quality/index.ts @@ -18,5 +18,6 @@ export default function ({ loadTestFile }: DatasetQualityFtrProviderContext) { loadTestFile(require.resolve('./degraded_field_flyout')); loadTestFile(require.resolve('./failed_docs_flyout')); loadTestFile(require.resolve('./home')); + loadTestFile(require.resolve('./dataset_quality_details_failure_store')); }); } diff --git a/x-pack/solutions/observability/test/functional/apps/dataset_quality/roles/role_management.ts b/x-pack/solutions/observability/test/functional/apps/dataset_quality/roles/role_management.ts index b6897ec1a234e..aa69523aa23cc 100644 --- a/x-pack/solutions/observability/test/functional/apps/dataset_quality/roles/role_management.ts +++ b/x-pack/solutions/observability/test/functional/apps/dataset_quality/roles/role_management.ts @@ -87,6 +87,27 @@ const datasetQualityRoles = { }, ], }, + canNotReadFailureStore: { + elasticsearch: { + cluster: ['monitor'], + indices: [ + { + names: ['logs-*'], + privileges: ['read'], + }, + ], + }, + kibana: [ + { + feature: { + dataQuality: ['minimal_all', 'manage_rules'], + discover: ['all'], + fleet: ['read'], + }, + spaces: ['*'], + }, + ], + }, }; const getDatasetQualityRole = ( diff --git a/x-pack/solutions/observability/test/functional/page_objects/dataset_quality.ts b/x-pack/solutions/observability/test/functional/page_objects/dataset_quality.ts index a11b0fcc3130b..221cfb473654d 100644 --- a/x-pack/solutions/observability/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/solutions/observability/test/functional/page_objects/dataset_quality.ts @@ -171,6 +171,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv 'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist', datasetQualityDetailsOverviewDegradedFieldToggleSwitch: 'datasetQualityDetailsOverviewDegradedFieldToggleSwitch', + datasetQualityDetailsSummaryCardFailedDocuments: + 'datasetQualityDetailsSummaryKpiCard-Failed documents', + datasetQualityDetailsSummaryCardNoFailureStore: + 'datasetQualityDetailsSummaryKpiCard-No failure store', + datasetQualityDetailsEnableFailureStoreButton: 'datasetQualityDetailsEnableFailureStoreButton', + editFailureStoreModal: 'editFailureStoreModal', + enableFailureStoreToggle: 'enableFailureStoreToggle', + failureStoreModalSaveButton: 'failureStoreModalSaveButton', }; return { diff --git a/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/data/logs_data.ts b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/data/logs_data.ts index cd1668484337e..79776555931cd 100644 --- a/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/data/logs_data.ts +++ b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/data/logs_data.ts @@ -221,3 +221,22 @@ export const ANOTHER_1024_CHARS = export const CONSISTENT_TAGS = [ 'this_is_here_to_remove_variance_introduced_by_the_geoip_processor', ]; + +export const processors = [ + { + geoip: { + field: 'host.ip', + target_field: 'geoip', + database_file: 'GeoLite2-City.mmdb', + properties: ['country_name', 'region_name', 'city_name'], + }, + }, + { + geoip: { + field: 'cloud.provider', + target_field: 'cloud.geoip', + database_file: 'GeoLite2-Country.mmdb', + properties: ['country_name'], + }, + }, +]; diff --git a/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/dataset_quality_details_failure_store.ts b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/dataset_quality_details_failure_store.ts new file mode 100644 index 0000000000000..1988715373e36 --- /dev/null +++ b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/dataset_quality_details_failure_store.ts @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { IndexTemplateName } from '@kbn/apm-synthtrace/src/lib/logs/custom_logsdb_index_templates'; +import type { FtrProviderContext } from '../../ftr_provider_context'; +import { getLogsForDataset, defaultNamespace, processors } from './data'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['datasetQuality', 'svlCommonPage']); + const testSubjects = getService('testSubjects'); + const synthtrace = getService('svlLogsSynthtraceClient'); + const to = '2024-01-01T12:00:00.000Z'; + + const failureStoreDatasetName = 'synth.2'; + const failureStoreDataStreamName = `logs-${failureStoreDatasetName}-${defaultNamespace}`; + + const noFailureStoreDatasetName = 'synth.no-fs'; + const noFailureStoreDataStreamName = `logs-${noFailureStoreDatasetName}-${defaultNamespace}`; + + describe('Dataset quality details failure store', function () { + // This disables the forward-compatibility test for Elasticsearch 8.19 with Kibana and ES 9.0. + // These versions are not expected to work together. Note: Failure store is not available in ES 9.0, + // and running these tests will result in an "unknown index privilege [read_failure_store]" error. + this.onlyEsVersion('8.19 || >=9.1'); + + before(async () => { + await synthtrace.createCustomPipeline(processors, 'synth.2@pipeline'); + await synthtrace.createCustomPipeline(processors, 'synth.no-fs@pipeline'); + await synthtrace.createComponentTemplate({ + name: 'synth.2@custom', + dataStreamOptions: { + failure_store: { + enabled: true, + }, + }, + }); + await synthtrace.createComponentTemplate({ + name: 'synth.no-fs@custom', + dataStreamOptions: { + failure_store: { + enabled: false, + }, + }, + }); + await synthtrace.createIndexTemplate(IndexTemplateName.Synht2); + await synthtrace.createIndexTemplate(IndexTemplateName.NoFailureStore); + + await synthtrace.index([ + // Index logs for synth.2 dataset + getLogsForDataset({ to, count: 5, dataset: failureStoreDatasetName }), + // Index logs for synth.no-fs dataset + getLogsForDataset({ to, count: 5, dataset: noFailureStoreDatasetName }), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await synthtrace.deleteIndexTemplate(IndexTemplateName.NoFailureStore); + await synthtrace.deleteComponentTemplate('synth.no-fs@custom'); + await synthtrace.deleteCustomPipeline('synth.no-fs@pipeline'); + await synthtrace.deleteIndexTemplate(IndexTemplateName.Synht2); + await synthtrace.deleteComponentTemplate('synth.2@custom'); + await synthtrace.deleteCustomPipeline('synth.2@pipeline'); + }); + + describe('without failure store permissions', () => { + before(async () => { + await PageObjects.svlCommonPage.loginAsViewer(); + }); + + it('should show "No failure store" card when failure store is disabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: noFailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + + it('should show "No failure store" card when failure store is enabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: failureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + }); + + describe('with failure store permissions', () => { + before(async () => { + await PageObjects.svlCommonPage.loginAsAdmin(); + }); + it('should show "No failure store" card when failure store is disabled', async () => { + const { datasetQualityDetailsSummaryCardNoFailureStore } = + PageObjects.datasetQuality.testSubjectSelectors; + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: noFailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardNoFailureStore + ); + expect(failedDocsCard).to.contain('No failure store'); + }); + + it('should open failure store modal and save new config', async () => { + const { + datasetQualityDetailsSummaryCardFailedDocuments, + datasetQualityDetailsSummaryCardNoFailureStore, + datasetQualityDetailsEnableFailureStoreButton, + editFailureStoreModal, + failureStoreModalSaveButton, + enableFailureStoreToggle, + } = PageObjects.datasetQuality.testSubjectSelectors; + + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: noFailureStoreDataStreamName, + }); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + await testSubjects.missingOrFail(datasetQualityDetailsSummaryCardFailedDocuments); + + await testSubjects.click(datasetQualityDetailsEnableFailureStoreButton); + + await testSubjects.existOrFail(editFailureStoreModal); + + const saveModalButton = await testSubjects.find(failureStoreModalSaveButton); + expect(await saveModalButton.isEnabled()).to.be(false); + + await testSubjects.click(enableFailureStoreToggle); + + expect(await saveModalButton.isEnabled()).to.be(true); + + await testSubjects.click(failureStoreModalSaveButton); + + await testSubjects.missingOrFail(editFailureStoreModal); + + await testSubjects.existOrFail(datasetQualityDetailsSummaryCardFailedDocuments); + + await testSubjects.missingOrFail(datasetQualityDetailsSummaryCardNoFailureStore); + + const failedDocsCard = await testSubjects.getVisibleText( + datasetQualityDetailsSummaryCardFailedDocuments + ); + expect(failedDocsCard).to.contain('Failed documents'); + }); + + it('should show failed docs count when failure store is enabled', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: failureStoreDataStreamName, + }); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsSummaryCardFailedDocuments + ); + const failedDocsCard = await testSubjects.getVisibleText( + PageObjects.datasetQuality.testSubjectSelectors + .datasetQualityDetailsSummaryCardFailedDocuments + ); + expect(failedDocsCard).to.not.contain('No failure store'); + }); + }); + }); +} diff --git a/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/index.ts b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/index.ts index d0ddfad0122cd..430460bcae556 100644 --- a/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/index.ts +++ b/x-pack/solutions/observability/test/serverless/functional/test_suites/dataset_quality/index.ts @@ -16,5 +16,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./dataset_quality_privileges')); loadTestFile(require.resolve('./dataset_quality_details')); loadTestFile(require.resolve('./degraded_field_flyout')); + loadTestFile(require.resolve('./dataset_quality_details_failure_store')); }); }