From 699aa4228b52648ef4362bde587a8337c494d3c2 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Thu, 12 Mar 2026 11:00:18 +0100 Subject: [PATCH 01/21] k8s otel flow --- .../kubernetes/data_ingest_status.tsx | 142 ++++++++------- .../quickstart_flows/kubernetes/index.tsx | 63 ++++++- .../build_install_stack_command.test.ts | 165 ++++++++++-------- .../build_install_stack_command.ts | 30 ++-- .../otel_kubernetes/otel_kubernetes_panel.tsx | 158 ++++++++--------- .../shared/get_started_panel.tsx | 15 +- .../server/routes/kubernetes/route.ts | 57 ++++-- 7 files changed, 368 insertions(+), 262 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index b8dd9660c2e1e..42edfb6f7e71d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -11,40 +11,37 @@ import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; -import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; -import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { ProgressIndicator } from '../shared/progress_indicator'; -import { GetStartedPanel } from '../shared/get_started_panel'; +import { GetStartedPanel, type ActionLink } from '../shared/get_started_panel'; import type { ObservabilityOnboardingContextValue } from '../../../plugin'; -import { usePricingFeature } from '../shared/use_pricing_feature'; -import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; -import { type IngestionMode } from '../shared/wired_streams_ingestion_selector'; -import { WIRED_ECS_DATA_VIEW_SPEC } from '../shared/wired_streams_data_view'; + +export type { ActionLink }; interface Props { onboardingId: string; - ingestionMode: IngestionMode; + onboardingFlowType: string; + dataset: string; + integration: string; + actionLinks: ActionLink[]; } const FETCH_INTERVAL = 2000; const SHOW_TROUBLESHOOTING_DELAY = 120000; // 2 minutes -const CLUSTER_OVERVIEW_DASHBOARD_ID = 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'; -export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { - const metricsOnboardingEnabled = usePricingFeature( - ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING - ); +export function DataIngestStatus({ + onboardingId, + onboardingFlowType, + dataset, + integration, + actionLinks, +}: Props) { const [checkDataStartTime] = useState(Date.now()); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); const { - services: { share, analytics }, + services: { analytics }, } = useKibana(); - const dashboardLocator = share.url.locators.get(DASHBOARD_APP_LOCATOR); - const logsLocator = share.url.locators.get(LOGS_LOCATOR_ID); - const useWiredStreams = ingestionMode === 'wired'; - const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_ECS_DATA_VIEW_SPEC } : {}; const { data, status, refetch } = useFetcher( (callApi) => { @@ -55,10 +52,18 @@ export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { [onboardingId] ); + const hasData = data?.hasData ?? false; + const hasLogs = data?.hasLogs ?? hasData; + const hasMetrics = data?.hasMetrics ?? hasData; + + const needsMetrics = actionLinks.some((actionLink) => actionLink.requires === 'metrics'); + useEffect(() => { const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(status) || data?.hasData === true) { + const isReady = needsMetrics ? hasMetrics : hasData; + + if (pendingStatusList.includes(status) || isReady) { return; } @@ -67,29 +72,69 @@ export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { }, FETCH_INTERVAL); return () => clearTimeout(timeout); - }, [data?.hasData, refetch, status]); + }, [hasData, hasMetrics, needsMetrics, refetch, status]); useEffect(() => { - if (data?.hasData === true && !dataReceivedTelemetrySent) { + if (hasData === true && !dataReceivedTelemetrySent) { setDataReceivedTelemetrySent(true); analytics.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { - flow_type: 'kubernetes', + flow_type: onboardingFlowType, flow_id: onboardingId, step: 'logs-ingest', step_status: 'complete', }); } - }, [analytics, data?.hasData, dataReceivedTelemetrySent, onboardingId]); + }, [analytics, hasData, dataReceivedTelemetrySent, onboardingFlowType, onboardingId]); const isTroubleshootingVisible = - data?.hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + + const filteredActionLinks = actionLinks.filter((actionLink) => { + const requires = actionLink.requires ?? 'any'; + + if (requires === 'logs') { + return hasLogs; + } + + if (requires === 'metrics') { + return hasMetrics; + } + + return hasData; + }); + + const filteredActionLinksWithHref = filteredActionLinks.filter((actionLink) => + Boolean(actionLink.href) + ); + + const progressTitle = (() => { + if (hasData && needsMetrics && !hasMetrics) { + return i18n.translate( + 'xpack.observability_onboarding.dataIngestStatus.waitingForMetricsTitle', + { defaultMessage: 'Waiting for metrics to be shipped' } + ); + } + + if (hasData) { + return i18n.translate( + 'xpack.observability_onboarding.dataIngestStatus.monitoringClusterTitle', + { + defaultMessage: 'We are monitoring your cluster', + } + ); + } + + return i18n.translate('xpack.observability_onboarding.dataIngestStatus.waitingForDataTitle', { + defaultMessage: 'Waiting for data to be shipped', + }); + })(); return ( <> )} - {data?.hasData === true && ( + {hasData === true && filteredActionLinksWithHref.length > 0 && ( <> )} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx index 3d95971034dc2..1132454f661a7 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx @@ -9,7 +9,10 @@ import React, { useEffect, useState } from 'react'; import type { EuiStepStatus } from '@elastic/eui'; import { EuiPanel, EuiSkeletonRectangle, EuiSkeletonText, EuiSpacer, EuiSteps } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; import { usePerformanceContext } from '@kbn/ebt-tools'; +import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; +import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; import { CommandSnippet } from './command_snippet'; @@ -19,6 +22,11 @@ import { useKubernetesFlow } from './use_kubernetes_flow'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { type IngestionMode } from '../shared/wired_streams_ingestion_selector'; +import { usePricingFeature } from '../shared/use_pricing_feature'; +import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; +import { WIRED_ECS_DATA_VIEW_SPEC } from '../shared/wired_streams_data_view'; +import type { ObservabilityOnboardingContextValue } from '../../../plugin'; +import type { ActionLink } from './data_ingest_status'; export const KubernetesPanel: React.FC = () => { useFlowBreadcrumb({ @@ -29,6 +37,16 @@ export const KubernetesPanel: React.FC = () => { const { data, status, error, refetch } = useKubernetesFlow(); const { onPageReady } = usePerformanceContext(); const [ingestionMode, setIngestionMode] = useState('classic'); + const metricsOnboardingEnabled = usePricingFeature( + ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING + ); + const { + services: { share }, + } = useKibana(); + const dashboardLocator = share.url.locators.get(DASHBOARD_APP_LOCATOR); + const logsLocator = share.url.locators.get(LOGS_LOCATOR_ID); + const useWiredStreams = ingestionMode === 'wired'; + const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_ECS_DATA_VIEW_SPEC } : {}; const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, @@ -50,6 +68,43 @@ export const KubernetesPanel: React.FC = () => { return ; } + const CLUSTER_OVERVIEW_DASHBOARD_ID = 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'; + const kubernetesActionLinks: ActionLink[] = [ + ...(metricsOnboardingEnabled + ? [ + { + id: CLUSTER_OVERVIEW_DASHBOARD_ID, + label: i18n.translate( + 'xpack.observability_onboarding.kubernetesPanel.exploreDashboard', + { defaultMessage: 'Explore Kubernetes cluster' } + ), + title: i18n.translate( + 'xpack.observability_onboarding.kubernetesPanel.monitoringCluster', + { + defaultMessage: 'Overview your Kubernetes cluster with this pre-made dashboard', + } + ), + requires: 'metrics' as const, + href: + dashboardLocator?.getRedirectUrl({ + dashboardId: CLUSTER_OVERVIEW_DASHBOARD_ID, + }) ?? '', + }, + ] + : []), + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.kubernetesPanel.logsTitle', { + defaultMessage: 'View and analyze your logs:', + }), + label: i18n.translate('xpack.observability_onboarding.kubernetesPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + requires: 'logs' as const, + href: logsLocator?.getRedirectUrl(logsLocatorParams) ?? '', + }, + ]; + const steps = [ { title: i18n.translate( @@ -90,7 +145,13 @@ export const KubernetesPanel: React.FC = () => { ), status: (isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive && data && ( - + ), }, ]; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts index 6ce9eb2490e79..422373fd728d2 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts @@ -10,28 +10,27 @@ import { OTEL_KUBE_STACK_VERSION, OTEL_STACK_NAMESPACE } from './constants'; import { buildInstallStackCommand } from './build_install_stack_command'; import { buildValuesFileUrl } from './build_values_file_url'; -describe('buildValuesFileUrl()', () => { - const isMetricsOnboardingEnabled = true; - +const TEST_ONBOARDING_ID = 'test-onboarding-id'; + +const defaultArgs = { + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: false, + managedOtlpEndpointUrl: 'https://example.com/otlp', + elasticsearchUrl: 'https://example.com/elasticsearch', + apiKeyEncoded: 'encoded_api_key', + agentVersion: '9.1.0', + onboardingId: TEST_ONBOARDING_ID, +} as const; + +describe('buildInstallStackCommand()', () => { it('builds command with Elasticsearch endpoint when OTLP service is not available', () => { - const isManagedOtlpServiceAvailable = false; - const managedOtlpEndpointUrl = 'https://example.com/otlp'; - const elasticsearchUrl = 'https://example.com/elasticsearch'; - const apiKeyEncoded = 'encoded_api_key'; - const agentVersion = '9.1.0'; + const { elasticsearchUrl, apiKeyEncoded, agentVersion } = defaultArgs; const otelKubeStackValuesFileUrl = buildValuesFileUrl({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - agentVersion, - }); - const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpEndpointUrl, - elasticsearchUrl, - apiKeyEncoded, + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: false, agentVersion, }); + const command = buildInstallStackCommand(defaultArgs); expect(command).toEqual(`kubectl create namespace ${OTEL_STACK_NAMESPACE} kubectl create secret generic elastic-secret-otel \\ @@ -41,27 +40,24 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'`); + --version '${OTEL_KUBE_STACK_VERSION}' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id'`); }); it('builds command with OTLP endpoint when OTLP service is available', () => { - const isManagedOtlpServiceAvailable = true; - const managedOtlpEndpointUrl = 'https://example.com/otlp'; - const elasticsearchUrl = 'https://example.com/elasticsearch'; - const apiKeyEncoded = 'encoded_api_key'; - const agentVersion = '9.1.0'; + const { managedOtlpEndpointUrl, apiKeyEncoded, agentVersion } = defaultArgs; const otelKubeStackValuesFileUrl = buildValuesFileUrl({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: true, agentVersion, }); const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpEndpointUrl, - elasticsearchUrl, - apiKeyEncoded, - agentVersion, + ...defaultArgs, + isManagedOtlpServiceAvailable: true, }); expect(command).toEqual(`kubectl create namespace ${OTEL_STACK_NAMESPACE} @@ -72,18 +68,50 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'`); + --version '${OTEL_KUBE_STACK_VERSION}' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id'`); + }); + + describe('onboarding_id processor', () => { + it('always injects resource/onboarding_id processor into logs pipeline', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + isMetricsOnboardingEnabled: false, + }); + + expect(command).toContain('resource\\/onboarding_id'); + expect(command).toContain(`onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}`); + expect(command).toContain( + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' + ); + }); + + it('injects resource/onboarding_id into metrics pipeline when metrics enabled', () => { + const command = buildInstallStackCommand(defaultArgs); + + expect(command).toContain( + 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id' + ); + }); + + it('does not inject resource/onboarding_id into metrics pipeline when metrics disabled', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + isMetricsOnboardingEnabled: false, + }); + + expect(command).not.toContain('metrics\\/node\\/otel'); + }); }); describe('wired streams', () => { it('does not include wired streams config when useWiredStreams is false', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: false, }); @@ -93,60 +121,46 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub it('routes daemon logs to wired streams when useWiredStreams is true (direct ES)', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, + ...defaultArgs, isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); expect(command).toContain('collectors.daemon.config.processors.resource\\/wired_streams'); expect(command).toContain('elasticsearch.index'); expect(command).toContain( - 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams' + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[9]=resource/wired_streams' ); expect(command).not.toContain('logs\\/apm'); }); it('routes daemon logs to wired streams when useWiredStreams is true (managed OTLP)', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, + ...defaultArgs, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); expect(command).toContain('collectors.daemon.config.processors.resource\\/wired_streams'); expect(command).toContain('elasticsearch.index'); expect(command).toContain( - 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams' + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[9]=resource/wired_streams' ); expect(command).not.toContain('logs\\/apm'); }); it('excludes APM logs from wired streams regardless of metrics onboarding setting', () => { const withMetrics = buildInstallStackCommand({ + ...defaultArgs, isMetricsOnboardingEnabled: true, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); const withoutMetrics = buildInstallStackCommand({ + ...defaultArgs, isMetricsOnboardingEnabled: false, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); @@ -156,12 +170,7 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub it('does not modify gateway config when useWiredStreams is true', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: true, }); @@ -169,21 +178,25 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub expect(command).not.toContain('logs_index=logs'); }); - it('appends wired streams config at the end of the helm command', () => { + it('assigns onboarding_id to logs processors[8] and wired_streams to logs processors[9]', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: true, }); - expect(command).toContain( - `--version '${OTEL_KUBE_STACK_VERSION}' \\ - --set 'collectors.daemon.config.processors.resource\\/wired_streams` - ); + expect(command).toContain('logs\\/node.processors[8]=resource/onboarding_id'); + expect(command).toContain('logs\\/node.processors[9]=resource/wired_streams'); + }); + + it('appends wired streams config after onboarding_id config', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + useWiredStreams: true, + }); + + const onboardingIdIndex = command.indexOf('resource/onboarding_id'); + const wiredStreamsIndex = command.indexOf('resource/wired_streams'); + expect(onboardingIdIndex).toBeLessThan(wiredStreamsIndex); }); }); }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts index 8a3bab765783c..d651723bd2e48 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts @@ -16,6 +16,7 @@ export function buildInstallStackCommand({ apiKeyEncoded, agentVersion, useWiredStreams = false, + onboardingId, }: { isMetricsOnboardingEnabled: boolean; isManagedOtlpServiceAvailable: boolean; @@ -24,6 +25,7 @@ export function buildInstallStackCommand({ apiKeyEncoded: string; agentVersion: string; useWiredStreams?: boolean; + onboardingId: string; }): string { const ingestEndpointUrl = isManagedOtlpServiceAvailable ? managedOtlpEndpointUrl @@ -37,22 +39,30 @@ export function buildInstallStackCommand({ agentVersion, }); + let nextLogProcessorIndex = 8; + let nextMetricProcessorIndex = 8; + + const onboardingIdConfig = (() => { + let config = ` \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${onboardingId}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[${nextLogProcessorIndex++}]=resource/onboarding_id'`; + if (isMetricsOnboardingEnabled) { + config += ` \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[${nextMetricProcessorIndex++}]=resource/onboarding_id'`; + } + return config; + })(); + const wiredStreamsConfig = (() => { if (!useWiredStreams) return ''; - // Route container logs to wired streams by injecting the - // resource/wired_streams processor on the daemon collector's node pipeline. - // APM logs are intentionally excluded — wired streams support for APM is - // tracked separately and not yet ready. - // K8s event logs (from the cluster collector) are unaffected and keep - // their classic routing (e.g. logs-k8sobjectsreceiver.otel-default). - // The elasticsearch.index resource attribute works for both direct ES - // (elasticsearch exporter) and managed OTLP (otlp/ingest exporter). return ` \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].action=upsert' \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].key=elasticsearch.index' \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].value=logs.otel' \\ - --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams'`; + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[${nextLogProcessorIndex++}]=resource/wired_streams'`; })(); return `kubectl create namespace ${OTEL_STACK_NAMESPACE} @@ -63,5 +73,5 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'${wiredStreamsConfig}`; + --version '${OTEL_KUBE_STACK_VERSION}'${onboardingIdConfig}${wiredStreamsConfig}`; } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx index ac1bf69602fe4..6ffc8474401cf 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx @@ -6,6 +6,7 @@ */ import React, { useState, useEffect } from 'react'; +import type { EuiStepStatus } from '@elastic/eui'; import { EuiPanel, EuiSkeletonText, @@ -30,9 +31,11 @@ import { usePerformanceContext } from '@kbn/ebt-tools'; import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; import { type ObservabilityOnboardingAppServices } from '../../..'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; -import { GetStartedPanel } from '../shared/get_started_panel'; import { FeedbackButtons } from '../shared/feedback_buttons'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { DataIngestStatus, type ActionLink } from '../kubernetes/data_ingest_status'; import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; import { WiredStreamsIngestionSelector, @@ -59,7 +62,7 @@ export const OtelKubernetesPanel: React.FC = () => { defaultMessage: 'Kubernetes: OpenTelemetry', }), }); - const { data, error, refetch } = useKubernetesFlow('kubernetes_otel'); + const { data, status, error, refetch } = useKubernetesFlow('kubernetes_otel'); const [idSelected, setIdSelected] = useState('nodejs'); const { services: { share, docLinks }, @@ -83,6 +86,12 @@ export const OtelKubernetesPanel: React.FC = () => { } = useWiredStreamsStatus(); const [ingestionMode, setIngestionMode] = useState('classic'); const useWiredStreams = ingestionMode === 'wired'; + + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS, + onboardingFlowType: 'kubernetes_otel', + onboardingId: data?.onboardingId, + }); const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_OTEL_DATA_VIEW_SPEC } : {}; useEffect(() => { @@ -118,9 +127,56 @@ export const OtelKubernetesPanel: React.FC = () => { apiKeyEncoded: data.apiKeyEncoded, agentVersion: data.elasticAgentVersionInfo.agentBaseVersion, useWiredStreams, + onboardingId: data.onboardingId, }) : undefined; + const otelKubernetesActionLinks: ActionLink[] = [ + ...(isMetricsOnboardingEnabled + ? [ + { + id: CLUSTER_OVERVIEW_DASHBOARD_ID, + title: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster', + { defaultMessage: 'Check your Kubernetes cluster health:' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.exploreDashboard', + { defaultMessage: 'Explore Kubernetes Cluster Dashboard' } + ), + requires: 'metrics' as const, + href: + dashboardLocator?.getRedirectUrl({ + dashboardId: CLUSTER_OVERVIEW_DASHBOARD_ID, + }) ?? '', + }, + { + id: 'services', + title: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.servicesTitle', + { defaultMessage: 'Check your application services:' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.servicesLabel', + { defaultMessage: 'Explore Service inventory' } + ), + requires: 'metrics' as const, + href: apmLocator?.getRedirectUrl({ serviceName: undefined }) ?? '', + }, + ] + : []), + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelKubernetesPanel.logsTitle', { + defaultMessage: 'View and analyze your logs:', + }), + label: i18n.translate('xpack.observability_onboarding.otelKubernetesPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: logsLocator?.getRedirectUrl(logsLocatorParams) ?? '', + }, + ]; + return ( @@ -227,7 +283,7 @@ export const OtelKubernetesPanel: React.FC = () => { 'xpack.observability_onboarding.otelKubernetesPanel.helmsAutogeneratedTLSCertificatesTextLabel', { defaultMessage: - "Helm's autogenerated TLS certificates have a default expiration period of 365 days. These certificates are not renewed automatically unless the release is manually updated. Enabling cert-manager allows for automatic certificate renewal.", + "Helm's autogenerated TLS certificates have a default expiration period of 365 days. These certificates are not renewed automatically unless the release is manually updated. Enabling cert-manager allows for automatic certificate renewal.", } )} position="top" @@ -443,93 +499,15 @@ kubectl describe pod -n my-namespace`} defaultMessage: 'Visualize your data', } ), - children: data ? ( - <> -

- {isMetricsOnboardingEnabled && ( - - )} - {!isMetricsOnboardingEnabled && ( - - )} -

- - - - ) : ( - + status: (isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive && data && ( + ), }, ]} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx index 1a1e2da18934e..7af267a9abd73 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx @@ -22,6 +22,14 @@ import type { OnboardingFlowEventContext } from '../../../../common/telemetry_ev import { OBSERVABILITY_ONBOARDING_FLOW_DATASET_DETECTED_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import type { ObservabilityOnboardingContextValue } from '../../../plugin'; +export interface ActionLink { + id: string; + title: string; + label: string; + href: string; + requires?: 'any' | 'logs' | 'metrics' | 'traces'; +} + export function GetStartedPanel({ onboardingFlowType, dataset, @@ -37,12 +45,7 @@ export function GetStartedPanel({ dataset: string; integration?: string; newTab: boolean; - actionLinks: Array<{ - id: string; - title: string; - label: string; - href: string; - }>; + actionLinks: ActionLink[]; previewImage?: string; isLoading: boolean; onboardingId?: string; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index ca969ff1b97bb..b8c62536f69e4 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -31,6 +31,8 @@ export interface CreateKubernetesOnboardingFlowRouteResponse { export interface HasKubernetesDataRouteResponse { hasData: boolean; + hasLogs?: boolean; + hasMetrics?: boolean; } const createKubernetesOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ @@ -132,22 +134,49 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ const { elasticsearch } = await resources.context.core; try { - const result = await elasticsearch.client.asCurrentUser.search({ - index: ['logs-*', 'metrics-*', 'logs', 'logs.*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query: { - bool: { - filter: termQuery('fields.onboarding_id', onboardingId), - }, + const query: estypes.QueryDslQueryContainer = { + bool: { + filter: [ + { + bool: { + should: [ + ...termQuery('fields.onboarding_id', onboardingId), + ...termQuery('resource.attributes.onboarding.id', onboardingId), + ...termQuery('labels.onboarding_id', onboardingId), + ], + minimum_should_match: 1, + }, + }, + ], }, - }); - const { value } = result.hits.total as estypes.SearchTotalHits; + }; + + const [logsResult, metricsResult] = await Promise.all([ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*', 'logs', 'logs.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*', 'metrics', 'metrics.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]); + + const hasLogs = (logsResult.hits.total as estypes.SearchTotalHits).value > 0; + const hasMetrics = (metricsResult.hits.total as estypes.SearchTotalHits).value > 0; return { - hasData: value > 0, + hasData: hasLogs || hasMetrics, + hasLogs, + hasMetrics, }; } catch (error) { const errorType = error?.meta?.body?.error?.type; @@ -159,6 +188,8 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ ) { return { hasData: false, + hasLogs: false, + hasMetrics: false, }; } From d73a7c4ffb9bba4881caab9daa507c12ae3b4660 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Fri, 13 Mar 2026 13:13:57 +0100 Subject: [PATCH 02/21] add otel host flow --- .../quickstart_flows/otel_logs/index.tsx | 276 +++++++++++------- .../server/routes/otel_host/route.ts | 69 +++++ 2 files changed, 244 insertions(+), 101 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index 9bbdbea3e4717..6d71a718c7240 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -18,19 +18,24 @@ import { EuiText, EuiButtonGroup, EuiCopy, - EuiImage, EuiCallOut, EuiSkeletonText, } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; +import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import useAsyncFn from 'react-use/lib/useAsyncFn'; import { FormattedMessage } from '@kbn/i18n-react'; import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { usePerformanceContext } from '@kbn/ebt-tools'; +import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; import type { ObservabilityOnboardingAppServices } from '../../..'; -import { useFetcher } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { ProgressIndicator } from '../shared/progress_indicator'; +import { GetStartedPanel } from '../shared/get_started_panel'; import { useWiredStreamsStatus } from '../../../hooks/use_wired_streams_status'; import { MultiIntegrationInstallBanner } from './multi_integration_install_banner'; import { EmptyPrompt } from '../shared/empty_prompt'; @@ -54,6 +59,9 @@ const HOST_COMMAND = i18n.translate( } ); +const FETCH_INTERVAL = 2000; +const SHOW_TROUBLESHOOTING_DELAY = 120_000; + export const OtelLogsPanel: React.FC = () => { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.autoDetectPanel.breadcrumbs.otelHost', { @@ -62,7 +70,7 @@ export const OtelLogsPanel: React.FC = () => { }); const { onPageReady } = usePerformanceContext(); const { - services: { share, http, docLinks }, + services: { share, docLinks, analytics }, } = useKibana(); const { @@ -87,6 +95,68 @@ export const OtelLogsPanel: React.FC = () => { } }, [onPageReady, setupData]); + const [sessionStartTime] = useState(() => new Date().toISOString()); + const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + isActive: !!setupData, + onboardingFlowType: 'otel_logs', + onboardingId: setupData?.onboardingId, + }); + + const [checkDataStartTime, setCheckDataStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && checkDataStartTime === null) { + setCheckDataStartTime(Date.now()); + } + }, [isMonitoringStepActive, checkDataStartTime]); + + const { + data: hasDataResponse, + status: hasDataStatus, + refetch: refetchHasData, + } = useFetcher( + (callApi) => { + if (!isMonitoringStepActive) return; + return callApi('GET /internal/observability_onboarding/otel_host/has-data', { + params: { + query: { start: sessionStartTime }, + }, + }); + }, + [isMonitoringStepActive, sessionStartTime], + { showToastOnError: false } + ); + + useEffect(() => { + const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; + if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { + return; + } + const timeout = setTimeout(() => { + refetchHasData(); + }, FETCH_INTERVAL); + return () => clearTimeout(timeout); + }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); + + useEffect(() => { + if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: 'otel_logs', + flow_id: setupData?.onboardingId ?? '', + step: 'logs-ingest', + step_status: 'complete', + }); + } + }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, setupData?.onboardingId]); + + const isTroubleshootingVisible = + isMonitoringStepActive && + hasDataResponse?.hasData === false && + checkDataStartTime !== null && + Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + const isMetricsOnboardingEnabled = usePricingFeature( ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING ); @@ -119,6 +189,45 @@ export const OtelLogsPanel: React.FC = () => { getDeeplinks(); }, [getDeeplinks]); + const visualizeActionLinks = useMemo( + () => + [ + ...(deeplinks?.logs + ? [ + { + id: 'logs', + title: i18n.translate( + 'xpack.observability_onboarding.otelLogsPanel.logsTitle', + { defaultMessage: 'View and analyze your logs' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelLogsPanel.logsLabel', + { defaultMessage: 'Explore logs' } + ), + href: deeplinks.logs, + }, + ] + : []), + ...(isMetricsOnboardingEnabled && deeplinks?.metrics + ? [ + { + id: 'metrics', + title: i18n.translate( + 'xpack.observability_onboarding.otelLogsPanel.metricsTitle', + { defaultMessage: 'View and analyze your metrics' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelLogsPanel.metricsLabel', + { defaultMessage: 'Open Hosts' } + ), + href: deeplinks.metrics, + }, + ] + : []), + ], + [deeplinks, isMetricsOnboardingEnabled] + ); + const installTabContents = useMemo( () => [ { @@ -359,111 +468,76 @@ export const OtelLogsPanel: React.FC = () => { defaultMessage: 'Visualize your data', } ), - children: ( + status: (hasDataResponse?.hasData + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive ? ( <> - -

- {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel', - { - defaultMessage: - 'After running the previous command, come back and view your data.', - } - )} -

- - - - - - - - - {deeplinks?.logs && ( - <> - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel', - { defaultMessage: 'View and analyze your logs' } - )} - - - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.exploreLogs', - { - defaultMessage: 'Explore logs', - } - )} - - - - )} - - {isMetricsOnboardingEnabled && deeplinks?.metrics && ( - <> - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel', - { defaultMessage: 'View and analyze your metrics' } - )} - - - + + + {isTroubleshootingVisible && ( + <> + + + {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.exploreMetrics', - { - defaultMessage: 'Open Hosts', - } + 'xpack.observability_onboarding.otelLogsPanel.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } )} - - - )} - - - - - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.documentationLink', - { - defaultMessage: 'Open documentation', - } - )} - - ), - }} - /> - + ), + }} + /> + + + )} + + {hasDataResponse?.hasData === true && visualizeActionLinks.length > 0 && ( + <> + + + + )} - ), + ) : null, }, ]} /> diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index f21efb35146d8..a3bb18483f27b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -5,7 +5,10 @@ * 2.0. */ +import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; @@ -25,6 +28,7 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ }, }, async handler(resources): Promise<{ + onboardingId: string; elasticAgentVersionInfo: ElasticAgentVersionInfo; elasticsearchUrl: string; apiKeyEncoded: string; @@ -65,6 +69,7 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ : await createShipperApiKey(client.asCurrentUser, `otel-host`); return { + onboardingId: uuidv4(), elasticsearchUrl: elasticsearchUrlList.length > 0 ? elasticsearchUrlList[0] : '', elasticAgentVersionInfo, apiKeyEncoded, @@ -73,6 +78,70 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ }, }); +const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/otel_host/has-data', + params: t.type({ + query: t.type({ + start: t.string, + }), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean }> { + const { start } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + try { + const query: estypes.QueryDslQueryContainer = { + bool: { + filter: [{ range: { '@timestamp': { gte: start } } }], + }, + }; + + const [logsResult, metricsResult] = await Promise.all([ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*.otel-*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]); + + const hasLogs = (logsResult.hits.total as estypes.SearchTotalHits).value > 0; + const hasMetrics = (metricsResult.hits.total as estypes.SearchTotalHits).value > 0; + + return { hasData: hasLogs || hasMetrics }; + } catch (error) { + const errorType = error?.meta?.body?.error?.type; + const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; + + if ( + errorType === 'search_phase_execution_exception' && + rootCauseType === 'no_shard_available_action_exception' + ) { + return { hasData: false }; + } + + throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + } + }, +}); + export const otelHostOnboardingRouteRepository = { ...setupFlowRoute, + ...hasOtelHostDataRoute, }; From 5384013b83de8591de7370a941b0ff70892f069f Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Fri, 13 Mar 2026 14:50:06 +0100 Subject: [PATCH 03/21] apm otel flow --- .../quickstart_flows/otel_apm/index.tsx | 187 ++++++++++++++---- .../server/routes/otel_apm/route.ts | 62 ++++++ 2 files changed, 212 insertions(+), 37 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index a19ec55977fa8..22bf4f8c1dab9 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -5,11 +5,12 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { css } from '@emotion/react'; import type { EuiBasicTableColumn } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; import { EuiBasicTable, EuiButtonIcon, @@ -26,8 +27,11 @@ import { import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { ValuesType } from 'utility-types'; import { usePerformanceContext } from '@kbn/ebt-tools'; -import { FETCH_STATUS } from '@kbn/observability-shared-plugin/public'; +import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import type { ObservabilityOnboardingAppServices } from '../../..'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { ProgressIndicator } from '../shared/progress_indicator'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { GetStartedPanel } from '../shared/get_started_panel'; @@ -35,6 +39,9 @@ import { ManagedOtlpCallout } from '../shared/managed_otlp_callout'; import { useOtelApmFlow } from './use_otel_apm_flow'; import { EmptyPrompt } from '../shared/empty_prompt'; +const FETCH_INTERVAL = 2000; +const SHOW_TROUBLESHOOTING_DELAY = 120_000; + export function OtelApmQuickstartFlow() { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.otelApm.breadcrumbs.k8sOtel', { @@ -42,7 +49,7 @@ export function OtelApmQuickstartFlow() { }), }); const { - services: { share }, + services: { share, analytics }, } = useKibana(); const { data, status, error, refetch } = useOtelApmFlow(); const { onPageReady } = usePerformanceContext(); @@ -57,6 +64,68 @@ export function OtelApmQuickstartFlow() { } }, [data, onPageReady]); + const [sessionStartTime] = useState(() => new Date().toISOString()); + const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS, + onboardingFlowType: 'otel_apm', + onboardingId: data?.onboardingId, + }); + + const [checkDataStartTime, setCheckDataStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && checkDataStartTime === null) { + setCheckDataStartTime(Date.now()); + } + }, [isMonitoringStepActive, checkDataStartTime]); + + const { + data: hasDataResponse, + status: hasDataStatus, + refetch: refetchHasData, + } = useFetcher( + (callApi) => { + if (!isMonitoringStepActive) return; + return callApi('GET /internal/observability_onboarding/otel_apm/has-data', { + params: { + query: { start: sessionStartTime }, + }, + }); + }, + [isMonitoringStepActive, sessionStartTime], + { showToastOnError: false } + ); + + useEffect(() => { + const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; + if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { + return; + } + const timeout = setTimeout(() => { + refetchHasData(); + }, FETCH_INTERVAL); + return () => clearTimeout(timeout); + }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); + + useEffect(() => { + if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: 'otel_apm', + flow_id: data?.onboardingId ?? '', + step: 'logs-ingest', + step_status: 'complete', + }); + } + }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, data?.onboardingId]); + + const isTroubleshootingVisible = + isMonitoringStepActive && + hasDataResponse?.hasData === false && + checkDataStartTime !== null && + Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + if (error !== undefined) { return ; } @@ -107,42 +176,86 @@ export function OtelApmQuickstartFlow() { title: i18n.translate('xpack.observability_onboarding.otelApm.monitorStepTitle', { defaultMessage: 'Visualize your data', }), - children: ( + status: (hasDataResponse?.hasData + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive ? ( <> -

- -

- - + + {isTroubleshootingVisible && ( + <> + + + + {i18n.translate( + 'xpack.observability_onboarding.otelApm.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } + )} + + ), + }} + /> + + + )} + + {hasDataResponse?.hasData === true && ( + <> + + + + )} - ), + ) : null, }, ]} /> @@ -173,7 +286,7 @@ function InstallSDKInstructions() { {i18n.translate('xpack.observability_onboarding.otelApm.EDOTDocumentationLinkLabel', { defaultMessage: 'Elastic Distribution of OpenTelemetry documentation', diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts index fef013c034a5a..21a4118eabba5 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts @@ -6,7 +6,9 @@ */ import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed_otlp_service_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; @@ -51,6 +53,66 @@ const createOtelApmOnboardingFlowRoute = createObservabilityOnboardingServerRout }, }); +const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/otel_apm/has-data', + params: t.type({ + query: t.type({ + start: t.string, + }), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean }> { + const { start } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + try { + const query: estypes.QueryDslQueryContainer = { + bool: { + filter: [{ range: { '@timestamp': { gte: start } } }], + }, + }; + + const result = await elasticsearch.client.asCurrentUser.search({ + index: [ + 'traces-apm*', + 'traces-*.otel-*', + 'logs-apm*', + 'logs-*.otel-*', + 'metrics-apm*', + 'metrics-*.otel-*', + 'apm-*', + ], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }); + + const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; + return { hasData }; + } catch (error) { + const errorType = error?.meta?.body?.error?.type; + const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; + + if ( + errorType === 'search_phase_execution_exception' && + rootCauseType === 'no_shard_available_action_exception' + ) { + return { hasData: false }; + } + + throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + } + }, +}); + export const otelApmOnboardingRouteRepository = { ...createOtelApmOnboardingFlowRoute, + ...hasOtelApmDataRoute, }; From 8032e28139bd1d4d8ec6f3a419132843f7579765 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Fri, 13 Mar 2026 16:39:08 +0100 Subject: [PATCH 04/21] cloud-forwarder --- .../common/aws_cloudforwarder.ts | 6 + .../quickstart_flows/cloudforwarder/index.tsx | 196 +++++++++++++++--- .../server/routes/cloudforwarder/route.ts | 59 ++++++ 3 files changed, 231 insertions(+), 30 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts index a236b06f9e9a3..7193d2c6efb9d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts @@ -27,3 +27,9 @@ export const CLOUDFORMATION_STACK_CONFIGS = { } as const; export type LogType = keyof typeof CLOUDFORMATION_STACK_CONFIGS; + +export const CLOUDFORWARDER_INDEX_PATTERNS: Record = { + vpcflow: 'logs-aws.vpcflow.otel-*', + elbaccess: 'logs-aws.elbaccess.otel-*', + cloudtrail: 'logs-aws.cloudtrail.otel-*', +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx index cd65f37d60612..b8b76f9811e62 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx @@ -6,10 +6,10 @@ */ import React, { useEffect, useState } from 'react'; +import { css } from '@emotion/react'; import { EuiButton, EuiButtonGroup, - EuiCallOut, EuiFieldText, EuiFormRow, EuiLink, @@ -20,23 +20,33 @@ import { EuiSteps, EuiText, } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { usePerformanceContext } from '@kbn/ebt-tools'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { ObservabilityOnboardingAppServices } from '../../..'; -import { FETCH_STATUS } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { ProgressIndicator } from '../shared/progress_indicator'; +import { GetStartedPanel } from '../shared/get_started_panel'; import { useCloudForwarderFlow } from './use_cloudforwarder_flow'; import { EmptyPrompt } from '../shared/empty_prompt'; -import { OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; +import { + OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT, + OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT, +} from '../../../../common/telemetry_events'; import { type LogType } from '../../../../common/aws_cloudforwarder'; import { isValidS3BucketName, buildS3BucketArn, buildCloudFormationUrl } from './utils'; const EDOT_CLOUD_FORWARDER_DOCS_URL = 'https://www.elastic.co/docs/reference/opentelemetry/edot-cloud-forwarder/aws'; +const FETCH_INTERVAL = 5000; +const SHOW_TROUBLESHOOTING_DELAY = 300_000; + export function CloudForwarderPanel() { useFlowBreadcrumb({ text: i18n.translate( @@ -49,6 +59,7 @@ export function CloudForwarderPanel() { const { services: { + http, analytics, context: { cloudServiceProvider }, }, @@ -62,6 +73,73 @@ export function CloudForwarderPanel() { trimmedBucketName.length > 0 && !isValidS3BucketName(trimmedBucketName); const { onPageReady } = usePerformanceContext(); + const [sessionStartTime, setSessionStartTime] = useState(null); + const [monitoringLogType, setMonitoringLogType] = useState(null); + const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const [launchStackClicked, setLaunchStackClicked] = useState(false); + + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS && launchStackClicked, + onboardingFlowType: 'cloudforwarder', + onboardingId: data?.onboardingId, + }); + + const [checkDataStartTime, setCheckDataStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && checkDataStartTime === null) { + setCheckDataStartTime(Date.now()); + } + }, [isMonitoringStepActive, checkDataStartTime]); + + const { + data: hasDataResponse, + status: hasDataStatus, + refetch: refetchHasData, + } = useFetcher( + (callApi) => { + if (!isMonitoringStepActive || !monitoringLogType || !sessionStartTime) return; + return callApi('GET /internal/observability_onboarding/cloudforwarder/has-data', { + params: { + query: { + logType: monitoringLogType, + start: sessionStartTime, + }, + }, + }); + }, + [isMonitoringStepActive, monitoringLogType, sessionStartTime], + { showToastOnError: false } + ); + + useEffect(() => { + const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; + if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { + return; + } + const timeout = setTimeout(() => { + refetchHasData(); + }, FETCH_INTERVAL); + return () => clearTimeout(timeout); + }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); + + useEffect(() => { + if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: 'cloudforwarder', + flow_id: data?.onboardingId ?? '', + step: 'logs-ingest', + step_status: 'complete', + }); + } + }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, data?.onboardingId]); + + const isTroubleshootingVisible = + isMonitoringStepActive && + hasDataResponse?.hasData === false && + checkDataStartTime !== null && + Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + useEffect(() => { if (data) { onPageReady({ @@ -216,6 +294,7 @@ export function CloudForwarderPanel() { value={s3BucketName} onChange={(e) => setS3BucketName(e.target.value)} isInvalid={isBucketNameInvalid} + disabled={launchStackClicked} placeholder={i18n.translate( 'xpack.observability_onboarding.cloudforwarderPanel.s3BucketNamePlaceholder', { @@ -246,6 +325,7 @@ export function CloudForwarderPanel() { idSelected={selectedLogType} onChange={(id) => setSelectedLogType(id as LogType)} buttonSize="m" + isDisabled={launchStackClicked} /> )} @@ -290,6 +370,9 @@ export function CloudForwarderPanel() { fill isDisabled={!isValidS3BucketName(trimmedBucketName)} onClick={() => { + setLaunchStackClicked(true); + setMonitoringLogType(selectedLogType); + setSessionStartTime(new Date().toISOString()); analytics?.reportEvent( OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT.eventType, { @@ -323,35 +406,88 @@ export function CloudForwarderPanel() { defaultMessage: 'Visualize your data', } ), - children: ( - + + + {isTroubleshootingVisible && ( + <> + + + + {i18n.translate( + 'xpack.observability_onboarding.cloudforwarderPanel.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } + )} + + ), + }} + /> + + )} - color="success" - iconType="check" - > -

- - {i18n.translate( - 'xpack.observability_onboarding.cloudforwarderPanel.strong.logsawsLabel', - { defaultMessage: 'logs-aws.*' } - )} - - ), - }} - /> -

- - ), + + {hasDataResponse?.hasData === true && ( + <> + + + + )} + + ) : null, }, ]; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts index 494b56051201a..96ea95724a38b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts @@ -6,11 +6,14 @@ */ import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed_otlp_service_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getManagedOtlpServiceUrl } from '../../lib/get_managed_otlp_service_url'; +import { CLOUDFORWARDER_INDEX_PATTERNS, type LogType } from '../../../common/aws_cloudforwarder'; export interface CreateCloudForwarderOnboardingFlowRouteResponse { onboardingId: string; @@ -58,6 +61,62 @@ const createCloudForwarderOnboardingFlowRoute = createObservabilityOnboardingSer }, }); +const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/cloudforwarder/has-data', + params: t.type({ + query: t.type({ + logType: t.string, + start: t.string, + }), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean }> { + const { logType, start } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + const indexPattern = CLOUDFORWARDER_INDEX_PATTERNS[logType as LogType]; + if (!indexPattern) { + throw Boom.badRequest(`Unknown logType: ${logType}`); + } + + try { + const result = await elasticsearch.client.asCurrentUser.search({ + index: [indexPattern], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: start } } }], + }, + }, + }); + + const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; + return { hasData }; + } catch (error) { + const errorType = error?.meta?.body?.error?.type; + const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; + + if ( + errorType === 'search_phase_execution_exception' && + rootCauseType === 'no_shard_available_action_exception' + ) { + return { hasData: false }; + } + + throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + } + }, +}); + export const cloudforwarderOnboardingRouteRepository = { ...createCloudForwarderOnboardingFlowRoute, + ...hasCloudForwarderDataRoute, }; From a433a8753bfbf1e523271bcba65d025190b5d024 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 17 Mar 2026 10:42:34 +0100 Subject: [PATCH 05/21] add runtime field for k8s logs essentials + wired flow --- .../server/routes/kubernetes/route.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index b8c62536f69e4..64736a83d1e60 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -151,6 +151,20 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; + // Logs Essentials + Wired Streams: logs.otel uses a passthrough mapping for + // resource.attributes, storing fields in _source without indexing them. The + // runtime field extracts the value at query time. On classic indexed streams + // the indexed mapping takes precedence and the runtime field is ignored. + const runtimeMappings: estypes.MappingRuntimeFields = { + 'resource.attributes.onboarding.id': { + type: 'keyword', + script: { + source: + "def v = params._source?.resource?.attributes?.get('onboarding.id'); if (v != null) emit(v.toString())", + }, + }, + }; + const [logsResult, metricsResult] = await Promise.all([ elasticsearch.client.asCurrentUser.search({ index: ['logs-*', 'logs', 'logs.*'], @@ -158,6 +172,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ allow_partial_search_results: true, size: 0, terminate_after: 1, + runtime_mappings: runtimeMappings, query, }), elasticsearch.client.asCurrentUser.search({ @@ -166,6 +181,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ allow_partial_search_results: true, size: 0, terminate_after: 1, + runtime_mappings: runtimeMappings, query, }), ]); From 4b99f827c3ce540614f70b7b27191bfd48cf51e4 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 17 Mar 2026 11:18:14 +0100 Subject: [PATCH 06/21] dedup + cleanup --- .../quickstart_flows/cloudforwarder/index.tsx | 83 +++------- .../quickstart_flows/kubernetes/index.tsx | 3 +- .../quickstart_flows/otel_apm/index.tsx | 76 ++------- .../build_install_stack_command.ts | 3 + .../quickstart_flows/otel_logs/index.tsx | 145 ++++++------------ .../shared/use_time_window_data_detection.ts | 98 ++++++++++++ .../lib/handle_has_data_search_error.ts | 31 ++++ .../server/routes/cloudforwarder/route.ts | 23 ++- .../server/routes/kubernetes/route.ts | 18 +-- .../server/routes/otel_apm/route.ts | 17 +- .../server/routes/otel_host/route.ts | 14 +- 11 files changed, 248 insertions(+), 263 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx index b8b76f9811e62..c62eb8ecf6de5 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx @@ -26,18 +26,16 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { usePerformanceContext } from '@kbn/ebt-tools'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { ObservabilityOnboardingAppServices } from '../../..'; -import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; import { ProgressIndicator } from '../shared/progress_indicator'; import { GetStartedPanel } from '../shared/get_started_panel'; import { useCloudForwarderFlow } from './use_cloudforwarder_flow'; import { EmptyPrompt } from '../shared/empty_prompt'; -import { - OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT, - OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT, -} from '../../../../common/telemetry_events'; +import { OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import { type LogType } from '../../../../common/aws_cloudforwarder'; import { isValidS3BucketName, buildS3BucketArn, buildCloudFormationUrl } from './utils'; @@ -75,7 +73,6 @@ export function CloudForwarderPanel() { const [sessionStartTime, setSessionStartTime] = useState(null); const [monitoringLogType, setMonitoringLogType] = useState(null); - const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); const [launchStackClicked, setLaunchStackClicked] = useState(false); const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ @@ -84,61 +81,17 @@ export function CloudForwarderPanel() { onboardingId: data?.onboardingId, }); - const [checkDataStartTime, setCheckDataStartTime] = useState(null); - useEffect(() => { - if (isMonitoringStepActive && checkDataStartTime === null) { - setCheckDataStartTime(Date.now()); - } - }, [isMonitoringStepActive, checkDataStartTime]); - - const { - data: hasDataResponse, - status: hasDataStatus, - refetch: refetchHasData, - } = useFetcher( - (callApi) => { - if (!isMonitoringStepActive || !monitoringLogType || !sessionStartTime) return; - return callApi('GET /internal/observability_onboarding/cloudforwarder/has-data', { - params: { - query: { - logType: monitoringLogType, - start: sessionStartTime, - }, - }, - }); - }, - [isMonitoringStepActive, monitoringLogType, sessionStartTime], - { showToastOnError: false } - ); - - useEffect(() => { - const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { - return; - } - const timeout = setTimeout(() => { - refetchHasData(); - }, FETCH_INTERVAL); - return () => clearTimeout(timeout); - }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); - - useEffect(() => { - if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { - setDataReceivedTelemetrySent(true); - analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { - flow_type: 'cloudforwarder', - flow_id: data?.onboardingId ?? '', - step: 'logs-ingest', - step_status: 'complete', - }); - } - }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, data?.onboardingId]); - - const isTroubleshootingVisible = - isMonitoringStepActive && - hasDataResponse?.hasData === false && - checkDataStartTime !== null && - Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: + isMonitoringStepActive && monitoringLogType !== null && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'cloudforwarder', + onboardingId: data?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/cloudforwarder/has-data', + extraQueryParams: monitoringLogType ? { logType: monitoringLogType } : undefined, + }); useEffect(() => { if (data) { @@ -406,7 +359,7 @@ export function CloudForwarderPanel() { defaultMessage: 'Visualize your data', } ), - status: (hasDataResponse?.hasData + status: (hasData ? 'complete' : isMonitoringStepActive ? 'current' @@ -415,7 +368,7 @@ export function CloudForwarderPanel() { <> )} - {hasDataResponse?.hasData === true && ( + {hasData === true && ( <> { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.autoDetectPanel.breadcrumbs.k8s', { @@ -68,7 +70,6 @@ export const KubernetesPanel: React.FC = () => { return ; } - const CLUSTER_OVERVIEW_DASHBOARD_ID = 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'; const kubernetesActionLinks: ActionLink[] = [ ...(metricsOnboardingEnabled ? [ diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index 22bf4f8c1dab9..5e4e0ea915ca1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -27,10 +27,10 @@ import { import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { ValuesType } from 'utility-types'; import { usePerformanceContext } from '@kbn/ebt-tools'; -import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import type { ObservabilityOnboardingAppServices } from '../../..'; -import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; import { ProgressIndicator } from '../shared/progress_indicator'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { FeedbackButtons } from '../shared/feedback_buttons'; @@ -49,7 +49,7 @@ export function OtelApmQuickstartFlow() { }), }); const { - services: { share, analytics }, + services: { share }, } = useKibana(); const { data, status, error, refetch } = useOtelApmFlow(); const { onPageReady } = usePerformanceContext(); @@ -65,7 +65,6 @@ export function OtelApmQuickstartFlow() { }, [data, onPageReady]); const [sessionStartTime] = useState(() => new Date().toISOString()); - const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, @@ -73,58 +72,15 @@ export function OtelApmQuickstartFlow() { onboardingId: data?.onboardingId, }); - const [checkDataStartTime, setCheckDataStartTime] = useState(null); - useEffect(() => { - if (isMonitoringStepActive && checkDataStartTime === null) { - setCheckDataStartTime(Date.now()); - } - }, [isMonitoringStepActive, checkDataStartTime]); - - const { - data: hasDataResponse, - status: hasDataStatus, - refetch: refetchHasData, - } = useFetcher( - (callApi) => { - if (!isMonitoringStepActive) return; - return callApi('GET /internal/observability_onboarding/otel_apm/has-data', { - params: { - query: { start: sessionStartTime }, - }, - }); - }, - [isMonitoringStepActive, sessionStartTime], - { showToastOnError: false } - ); - - useEffect(() => { - const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { - return; - } - const timeout = setTimeout(() => { - refetchHasData(); - }, FETCH_INTERVAL); - return () => clearTimeout(timeout); - }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); - - useEffect(() => { - if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { - setDataReceivedTelemetrySent(true); - analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { - flow_type: 'otel_apm', - flow_id: data?.onboardingId ?? '', - step: 'logs-ingest', - step_status: 'complete', - }); - } - }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, data?.onboardingId]); - - const isTroubleshootingVisible = - isMonitoringStepActive && - hasDataResponse?.hasData === false && - checkDataStartTime !== null && - Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: isMonitoringStepActive, + sessionStartTime, + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'otel_apm', + onboardingId: data?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/otel_apm/has-data', + }); if (error !== undefined) { return ; @@ -176,7 +132,7 @@ export function OtelApmQuickstartFlow() { title: i18n.translate('xpack.observability_onboarding.otelApm.monitorStepTitle', { defaultMessage: 'Visualize your data', }), - status: (hasDataResponse?.hasData + status: (hasData ? 'complete' : isMonitoringStepActive ? 'current' @@ -185,7 +141,7 @@ export function OtelApmQuickstartFlow() { <> )} - {hasDataResponse?.hasData === true && ( + {hasData === true && ( <> { }); const { onPageReady } = usePerformanceContext(); const { - services: { share, docLinks, analytics }, + services: { share, docLinks }, } = useKibana(); const { @@ -96,7 +96,6 @@ export const OtelLogsPanel: React.FC = () => { }, [onPageReady, setupData]); const [sessionStartTime] = useState(() => new Date().toISOString()); - const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: !!setupData, @@ -104,58 +103,15 @@ export const OtelLogsPanel: React.FC = () => { onboardingId: setupData?.onboardingId, }); - const [checkDataStartTime, setCheckDataStartTime] = useState(null); - useEffect(() => { - if (isMonitoringStepActive && checkDataStartTime === null) { - setCheckDataStartTime(Date.now()); - } - }, [isMonitoringStepActive, checkDataStartTime]); - - const { - data: hasDataResponse, - status: hasDataStatus, - refetch: refetchHasData, - } = useFetcher( - (callApi) => { - if (!isMonitoringStepActive) return; - return callApi('GET /internal/observability_onboarding/otel_host/has-data', { - params: { - query: { start: sessionStartTime }, - }, - }); - }, - [isMonitoringStepActive, sessionStartTime], - { showToastOnError: false } - ); - - useEffect(() => { - const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { - return; - } - const timeout = setTimeout(() => { - refetchHasData(); - }, FETCH_INTERVAL); - return () => clearTimeout(timeout); - }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus]); - - useEffect(() => { - if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { - setDataReceivedTelemetrySent(true); - analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { - flow_type: 'otel_logs', - flow_id: setupData?.onboardingId ?? '', - step: 'logs-ingest', - step_status: 'complete', - }); - } - }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, setupData?.onboardingId]); - - const isTroubleshootingVisible = - isMonitoringStepActive && - hasDataResponse?.hasData === false && - checkDataStartTime !== null && - Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: isMonitoringStepActive, + sessionStartTime, + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'otel_logs', + onboardingId: setupData?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/otel_host/has-data', + }); const isMetricsOnboardingEnabled = usePricingFeature( ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING @@ -190,41 +146,36 @@ export const OtelLogsPanel: React.FC = () => { }, [getDeeplinks]); const visualizeActionLinks = useMemo( - () => - [ - ...(deeplinks?.logs - ? [ - { - id: 'logs', - title: i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.logsTitle', - { defaultMessage: 'View and analyze your logs' } - ), - label: i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.logsLabel', - { defaultMessage: 'Explore logs' } - ), - href: deeplinks.logs, - }, - ] - : []), - ...(isMetricsOnboardingEnabled && deeplinks?.metrics - ? [ - { - id: 'metrics', - title: i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.metricsTitle', - { defaultMessage: 'View and analyze your metrics' } - ), - label: i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.metricsLabel', - { defaultMessage: 'Open Hosts' } - ), - href: deeplinks.metrics, - }, - ] - : []), - ], + () => [ + ...(deeplinks?.logs + ? [ + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { + defaultMessage: 'View and analyze your logs', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: deeplinks.logs, + }, + ] + : []), + ...(isMetricsOnboardingEnabled && deeplinks?.metrics + ? [ + { + id: 'metrics', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { + defaultMessage: 'View and analyze your metrics', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { + defaultMessage: 'Open Hosts', + }), + href: deeplinks.metrics, + }, + ] + : []), + ], [deeplinks, isMetricsOnboardingEnabled] ); @@ -468,16 +419,16 @@ export const OtelLogsPanel: React.FC = () => { defaultMessage: 'Visualize your data', } ), - status: (hasDataResponse?.hasData + status: (hasData ? 'complete' : isMonitoringStepActive - ? 'current' - : 'incomplete') as EuiStepStatus, + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> { ) } iconType="checkInCircleFilled" - isLoading={!hasDataResponse?.hasData} + isLoading={!hasData} css={css` max-width: 40%; `} @@ -522,7 +473,7 @@ export const OtelLogsPanel: React.FC = () => { )} - {hasDataResponse?.hasData === true && visualizeActionLinks.length > 0 && ( + {hasData === true && visualizeActionLinks.length > 0 && ( <> ; +} + +export function useTimeWindowDataDetection({ + isMonitoringActive, + sessionStartTime, + fetchInterval, + troubleshootingDelay, + flowType, + onboardingId, + endpoint, + extraQueryParams, +}: UseTimeWindowDataDetectionOptions) { + const [checkDataStartTime, setCheckDataStartTime] = useState(null); + const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const { + services: { analytics }, + } = useKibana(); + + useEffect(() => { + if (isMonitoringActive && checkDataStartTime === null) { + setCheckDataStartTime(Date.now()); + } + }, [isMonitoringActive, checkDataStartTime]); + + const { + data: hasDataResponse, + status: hasDataStatus, + refetch: refetchHasData, + } = useFetcher( + (callApi) => { + if (!isMonitoringActive) return; + return callApi(`GET ${endpoint}` as any, { + params: { + query: { start: sessionStartTime, ...extraQueryParams }, + }, + }); + }, + + [isMonitoringActive, sessionStartTime, endpoint, JSON.stringify(extraQueryParams)], + { showToastOnError: false } + ); + + useEffect(() => { + const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; + if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { + return; + } + const timeout = setTimeout(() => { + refetchHasData(); + }, fetchInterval); + return () => clearTimeout(timeout); + }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus, fetchInterval]); + + useEffect(() => { + if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: flowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'complete', + }); + } + }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, flowType, onboardingId]); + + const isTroubleshootingVisible = + isMonitoringActive && + hasDataResponse?.hasData === false && + checkDataStartTime !== null && + Date.now() - checkDataStartTime > troubleshootingDelay; + + return { + hasData: hasDataResponse?.hasData ?? false, + isTroubleshootingVisible, + }; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts new file mode 100644 index 0000000000000..ac7abb2720c5e --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; + +/** + * Handles errors from has-data Elasticsearch search requests. + * Returns true if the error is a transient "no shards available" condition + * (which should be treated as hasData: false), or throws a Boom error otherwise. + */ +export function isNoShardsAvailableError(error: any): boolean { + const errorType = error?.meta?.body?.error?.type; + const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; + + if ( + errorType === 'search_phase_execution_exception' && + rootCauseType === 'no_shard_available_action_exception' + ) { + return true; + } + + return false; +} + +export function throwHasDataSearchError(error: any): never { + throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts index 96ea95724a38b..2aface3e84603 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts @@ -13,7 +13,11 @@ import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getManagedOtlpServiceUrl } from '../../lib/get_managed_otlp_service_url'; -import { CLOUDFORWARDER_INDEX_PATTERNS, type LogType } from '../../../common/aws_cloudforwarder'; +import { CLOUDFORWARDER_INDEX_PATTERNS } from '../../../common/aws_cloudforwarder'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; export interface CreateCloudForwarderOnboardingFlowRouteResponse { onboardingId: string; @@ -65,7 +69,7 @@ const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/cloudforwarder/has-data', params: t.type({ query: t.type({ - logType: t.string, + logType: t.keyof({ vpcflow: null, elbaccess: null, cloudtrail: null }), start: t.string, }), }), @@ -79,10 +83,7 @@ const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ const { logType, start } = resources.params.query; const { elasticsearch } = await resources.context.core; - const indexPattern = CLOUDFORWARDER_INDEX_PATTERNS[logType as LogType]; - if (!indexPattern) { - throw Boom.badRequest(`Unknown logType: ${logType}`); - } + const indexPattern = CLOUDFORWARDER_INDEX_PATTERNS[logType]; try { const result = await elasticsearch.client.asCurrentUser.search({ @@ -101,17 +102,11 @@ const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; return { hasData }; } catch (error) { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; - - if ( - errorType === 'search_phase_execution_exception' && - rootCauseType === 'no_shard_available_action_exception' - ) { + if (isNoShardsAvailableError(error)) { return { hasData: false }; } - throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + throwHasDataSearchError(error); } }, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index 64736a83d1e60..bb0817b3973d0 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -10,6 +10,10 @@ import * as t from 'io-ts'; import Boom from '@hapi/boom'; import { termQuery } from '@kbn/observability-plugin/server'; import type { estypes } from '@elastic/elasticsearch'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; @@ -167,7 +171,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ const [logsResult, metricsResult] = await Promise.all([ elasticsearch.client.asCurrentUser.search({ - index: ['logs-*', 'logs', 'logs.*'], + index: ['logs-*', 'logs.*'], ignore_unavailable: true, allow_partial_search_results: true, size: 0, @@ -176,7 +180,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ query, }), elasticsearch.client.asCurrentUser.search({ - index: ['metrics-*', 'metrics', 'metrics.*'], + index: ['metrics-*', 'metrics.*'], ignore_unavailable: true, allow_partial_search_results: true, size: 0, @@ -195,13 +199,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ hasMetrics, }; } catch (error) { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; - - if ( - errorType === 'search_phase_execution_exception' && - rootCauseType === 'no_shard_available_action_exception' - ) { + if (isNoShardsAvailableError(error)) { return { hasData: false, hasLogs: false, @@ -209,7 +207,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }; } - throw Boom.internal(`Elasticsearch responses with an error. ${error.message}`); + throwHasDataSearchError(error); } }, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts index 21a4118eabba5..f279b9712fdf3 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts @@ -13,6 +13,10 @@ import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getManagedOtlpServiceUrl } from '../../lib/get_managed_otlp_service_url'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; const createOtelApmOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ endpoint: 'POST /internal/observability_onboarding/otel_apm/flow', @@ -77,6 +81,9 @@ const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ }, }; + // Time-window detection: matches any APM data arriving after session start. + // May produce false positives if other services are already active. + // Correlation ID fallback (labels.onboarding_id) is available if needed. const result = await elasticsearch.client.asCurrentUser.search({ index: [ 'traces-apm*', @@ -97,17 +104,11 @@ const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; return { hasData }; } catch (error) { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; - - if ( - errorType === 'search_phase_execution_exception' && - rootCauseType === 'no_shard_available_action_exception' - ) { + if (isNoShardsAvailableError(error)) { return { hasData: false }; } - throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + throwHasDataSearchError(error); } }, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index a3bb18483f27b..17c8a907e6ce7 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -12,6 +12,10 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; import { getAgentVersionInfo } from '../../lib/get_agent_version'; import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; @@ -126,17 +130,11 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ return { hasData: hasLogs || hasMetrics }; } catch (error) { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; - - if ( - errorType === 'search_phase_execution_exception' && - rootCauseType === 'no_shard_available_action_exception' - ) { + if (isNoShardsAvailableError(error)) { return { hasData: false }; } - throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); + throwHasDataSearchError(error); } }, }); From 3bd3dbd5048c50cb5f07f04fd9e0b6fe4d280dff Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 17 Mar 2026 15:31:19 +0100 Subject: [PATCH 07/21] cleanup --- .../kubernetes/data_ingest_status.tsx | 16 +++++++++--- .../quickstart_flows/kubernetes/index.tsx | 9 ++++++- .../quickstart_flows/otel_apm/index.tsx | 25 +++++++++++++++---- .../otel_kubernetes/otel_kubernetes_panel.tsx | 9 ++++++- .../shared/use_time_window_data_detection.ts | 22 ++++++++++++---- .../server/routes/kubernetes/route.ts | 10 +++++--- 6 files changed, 72 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index 42edfb6f7e71d..e8032045cac22 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -25,6 +25,7 @@ interface Props { dataset: string; integration: string; actionLinks: ActionLink[]; + onDataReceived?: () => void; } const FETCH_INTERVAL = 2000; @@ -36,6 +37,7 @@ export function DataIngestStatus({ dataset, integration, actionLinks, + onDataReceived, }: Props) { const [checkDataStartTime] = useState(Date.now()); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); @@ -57,12 +59,11 @@ export function DataIngestStatus({ const hasMetrics = data?.hasMetrics ?? hasData; const needsMetrics = actionLinks.some((actionLink) => actionLink.requires === 'metrics'); + const isReady = needsMetrics ? hasMetrics : hasData; useEffect(() => { const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - const isReady = needsMetrics ? hasMetrics : hasData; - if (pendingStatusList.includes(status) || isReady) { return; } @@ -72,7 +73,7 @@ export function DataIngestStatus({ }, FETCH_INTERVAL); return () => clearTimeout(timeout); - }, [hasData, hasMetrics, needsMetrics, refetch, status]); + }, [isReady, refetch, status]); useEffect(() => { if (hasData === true && !dataReceivedTelemetrySent) { @@ -86,6 +87,15 @@ export function DataIngestStatus({ } }, [analytics, hasData, dataReceivedTelemetrySent, onboardingFlowType, onboardingId]); + // Notify parent when all required data types have arrived (not just any data). + // This drives the step status to 'complete' and must wait for metrics + // if any action link requires them. + useEffect(() => { + if (isReady) { + onDataReceived?.(); + } + }, [isReady, onDataReceived]); + const isTroubleshootingVisible = hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx index 5c2ba85d7c2be..b4ca9dad5204d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx @@ -50,6 +50,8 @@ export const KubernetesPanel: React.FC = () => { const useWiredStreams = ingestionMode === 'wired'; const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_ECS_DATA_VIEW_SPEC } : {}; + const [dataReceived, setDataReceived] = useState(false); + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'kubernetes', @@ -144,7 +146,11 @@ export const KubernetesPanel: React.FC = () => { defaultMessage: 'Monitor your Kubernetes cluster', } ), - status: (isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, + status: (dataReceived + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive && data && ( { dataset="kubernetes" integration="kubernetes" actionLinks={kubernetesActionLinks} + onDataReceived={() => setDataReceived(true)} /> ), }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index 5e4e0ea915ca1..a23d61469c32b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -64,17 +64,25 @@ export function OtelApmQuickstartFlow() { } }, [data, onPageReady]); - const [sessionStartTime] = useState(() => new Date().toISOString()); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'otel_apm', onboardingId: data?.onboardingId, }); + // Set sessionStartTime when monitoring begins (first blur) rather than on + // mount, to narrow the time-window and reduce false positives from other + // APM services already ingesting data on the same cluster. + const [sessionStartTime, setSessionStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && sessionStartTime === null) { + setSessionStartTime(new Date().toISOString()); + } + }, [isMonitoringStepActive, sessionStartTime]); + const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ - isMonitoringActive: isMonitoringStepActive, - sessionStartTime, + isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', fetchInterval: FETCH_INTERVAL, troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, flowType: 'otel_apm', @@ -342,7 +350,14 @@ function ConfigureSDKInstructions({ - + ); } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx index 6ffc8474401cf..26a0c171ac773 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx @@ -87,6 +87,8 @@ export const OtelKubernetesPanel: React.FC = () => { const [ingestionMode, setIngestionMode] = useState('classic'); const useWiredStreams = ingestionMode === 'wired'; + const [dataReceived, setDataReceived] = useState(false); + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'kubernetes_otel', @@ -499,7 +501,11 @@ kubectl describe pod -n my-namespace`} defaultMessage: 'Visualize your data', } ), - status: (isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, + status: (dataReceived + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive && data && ( -n my-namespace`} dataset="kubernetes" integration="kubernetes_otel" actionLinks={otelKubernetesActionLinks} + onDataReceived={() => setDataReceived(true)} /> ), }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts index 46f3bd40123d5..83b4a469e1886 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; @@ -44,6 +44,12 @@ export function useTimeWindowDataDetection({ } }, [isMonitoringActive, checkDataStartTime]); + const stableExtraQueryParams = useMemo( + () => extraQueryParams, + // eslint-disable-next-line react-hooks/exhaustive-deps + [JSON.stringify(extraQueryParams)] + ); + const { data: hasDataResponse, status: hasDataStatus, @@ -53,12 +59,11 @@ export function useTimeWindowDataDetection({ if (!isMonitoringActive) return; return callApi(`GET ${endpoint}` as any, { params: { - query: { start: sessionStartTime, ...extraQueryParams }, + query: { start: sessionStartTime, ...stableExtraQueryParams }, }, }); }, - - [isMonitoringActive, sessionStartTime, endpoint, JSON.stringify(extraQueryParams)], + [isMonitoringActive, sessionStartTime, endpoint, stableExtraQueryParams], { showToastOnError: false } ); @@ -85,9 +90,16 @@ export function useTimeWindowDataDetection({ } }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, flowType, onboardingId]); + // Treat both "hasData === false" and fetch failures (where hasDataResponse + // is undefined but the request completed) as "no data yet" for the purpose + // of showing troubleshooting guidance. Without this, persistent fetch + // errors would leave the UI in a "waiting forever" state. + const noDataConfirmed = + hasDataResponse?.hasData === false || hasDataStatus === FETCH_STATUS.FAILURE; + const isTroubleshootingVisible = isMonitoringActive && - hasDataResponse?.hasData === false && + noDataConfirmed && checkDataStartTime !== null && Date.now() - checkDataStartTime > troubleshootingDelay; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index bb0817b3973d0..4b012f46dcb95 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -146,6 +146,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ should: [ ...termQuery('fields.onboarding_id', onboardingId), ...termQuery('resource.attributes.onboarding.id', onboardingId), + ...termQuery('resource.attributes.onboarding.id._rt', onboardingId), ...termQuery('labels.onboarding_id', onboardingId), ], minimum_should_match: 1, @@ -156,11 +157,12 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }; // Logs Essentials + Wired Streams: logs.otel uses a passthrough mapping for - // resource.attributes, storing fields in _source without indexing them. The - // runtime field extracts the value at query time. On classic indexed streams - // the indexed mapping takes precedence and the runtime field is ignored. + // resource.attributes, storing fields in _source without indexing them. + // We use a distinct runtime field name so it does not shadow the indexed + // mapping on classic streams where resource.attributes.onboarding.id is + // already a keyword. The query includes both names in the should clause. const runtimeMappings: estypes.MappingRuntimeFields = { - 'resource.attributes.onboarding.id': { + 'resource.attributes.onboarding.id._rt': { type: 'keyword', script: { source: From 1cace10e03f84b78c0f65c697c7687941cac4878 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Wed, 18 Mar 2026 10:36:01 +0100 Subject: [PATCH 08/21] update ensemble tests --- .../e2e/playwright/stateful/host_otel.spec.ts | 21 +++++++++++++----- .../stateful/kubernetes_otel.spec.ts | 22 ++++++++++++++----- .../stateful/pom/pages/otel_host_flow.page.ts | 19 ++++++++++++++-- .../pom/pages/otel_kubernetes_flow.page.ts | 11 ++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts index 61c2aa711b25f..41543157122c0 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts @@ -57,12 +57,23 @@ test('Otel Host', async ({ fs.writeFileSync(outputPath, codeSnippet); /** - * There is no explicit data ingest indication - * in the flow, so we need to rely on a timeout. - * 3 minutes should be enough for the collector - * to initialize and start ingesting data. + * The page waits for the browser window to lose + * focus as a signal to start checking for incoming data */ - await page.waitForTimeout(3 * 60000); + await page.evaluate('window.dispatchEvent(new Event("blur"))'); + + /** + * Wait for the data received indicator to appear. + * The flow polls for data after the blur event and + * shows "We are monitoring your host" once data arrives. + */ + await otelHostFlowPage.assertDataReceivedIndicator(); + + /** + * Additional buffer to ensure data has propagated + * to dashboards and Discover before navigating. + */ + await page.waitForTimeout(2 * 60000); /** * Wired streams only reroutes logs (to logs.otel); metrics and traces are diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts index 2247b3c29bb4d..e5a391d3c682e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts @@ -89,12 +89,24 @@ test('Otel Kubernetes', async ({ fs.writeFileSync(outputPath, codeSnippet); /** - * There is no explicit data ingest indication - * in the flow, so we need to rely on a timeout. - * 5 minutes should be enough for the stack to be - * created and to start pushing data. + * The page waits for the browser window to lose + * focus as a signal to start checking for incoming data */ - await page.waitForTimeout(5 * 60000); + await page.evaluate('window.dispatchEvent(new Event("blur"))'); + + /** + * Wait for the data received indicator to appear. + * The flow now uses DataIngestStatus which polls for data + * after the blur event and shows "We are monitoring your cluster" + * once both logs and metrics have arrived. + */ + await otelKubernetesFlowPage.assertDataReceivedIndicator(); + + /** + * Additional buffer to ensure data has propagated + * to dashboards and Discover before navigating. + */ + await page.waitForTimeout(2 * 60000); /** * Wired streams only reroutes logs (to logs.otel); metrics and traces are diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts index 339ad7da04160..27d64a4e2143f 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts @@ -12,12 +12,20 @@ export class OtelHostFlowPage { private readonly exploreLogsButton: Locator; private readonly exploreMetricsButton: Locator; + private readonly dataReceivedIndicator: Locator; constructor(page: Page) { this.page = page; - this.exploreLogsButton = this.page.getByTestId('obltOnboardingExploreLogs'); - this.exploreMetricsButton = this.page.getByTestId('obltOnboardingExploreMetrics'); + this.exploreLogsButton = this.page.getByTestId( + 'observabilityOnboardingDataIngestStatusActionLink-logs' + ); + this.exploreMetricsButton = this.page.getByTestId( + 'observabilityOnboardingDataIngestStatusActionLink-metrics' + ); + this.dataReceivedIndicator = this.page + .getByTestId('observabilityOnboardingOtelHostDataProgressIndicator') + .getByText('We are monitoring your host'); } public async selectPlatform(osName: string) { @@ -56,6 +64,13 @@ export class OtelHostFlowPage { await this.exploreLogsButton.click(); } + public async assertDataReceivedIndicator() { + await expect( + this.dataReceivedIndicator, + 'Data received indicator should be visible' + ).toBeVisible(); + } + public async assertLogsExplorationButtonVisible() { await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible(); } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts index 3828dacef0caa..689d1ae2be437 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts @@ -12,12 +12,16 @@ export class OtelKubernetesFlowPage { context: BrowserContext; private readonly exploreLogsButton: Locator; + private readonly dataReceivedIndicator: Locator; constructor(page: Page, context: BrowserContext) { this.page = page; this.context = context; this.exploreLogsButton = this.page.getByText('Explore logs'); + this.dataReceivedIndicator = this.page + .getByTestId('observabilityOnboardingKubernetesPanelDataProgressIndicator') + .getByText('We are monitoring your cluster'); } public async copyHelmRepositorySnippetToClipboard() { @@ -82,6 +86,13 @@ export class OtelKubernetesFlowPage { } } + public async assertDataReceivedIndicator() { + await expect( + this.dataReceivedIndicator, + 'Data received indicator should be visible' + ).toBeVisible(); + } + public async assertLogsExplorationButtonVisible() { await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible(); } From 384d8c746f679b3dd8f62696f513a3c0017df924 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Thu, 19 Mar 2026 12:12:23 +0100 Subject: [PATCH 09/21] harden data detection --- .../quickstart_flows/otel_logs/index.tsx | 164 ++++++++++-------- .../shared/use_time_window_data_detection.ts | 21 ++- .../server/routes/otel_host/route.ts | 16 +- 3 files changed, 118 insertions(+), 83 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index df87d496ad41e..f1b07bc1b99b2 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -62,6 +62,13 @@ const HOST_COMMAND = i18n.translate( const FETCH_INTERVAL = 2000; const SHOW_TROUBLESHOOTING_DELAY = 120_000; +// Used to scope time-window detection to the user's OS and avoid false positives from other hosts. +const OS_TYPE_MAP: Record = { + linux: 'linux', + mac: 'darwin', + windows: 'windows', +}; + export const OtelLogsPanel: React.FC = () => { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.autoDetectPanel.breadcrumbs.otelHost', { @@ -95,7 +102,7 @@ export const OtelLogsPanel: React.FC = () => { } }, [onPageReady, setupData]); - const [sessionStartTime] = useState(() => new Date().toISOString()); + const [selectedTab, setSelectedTab] = useState('linux'); const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ isActive: !!setupData, @@ -103,14 +110,25 @@ export const OtelLogsPanel: React.FC = () => { onboardingId: setupData?.onboardingId, }); + // Set sessionStartTime when monitoring begins (first blur) rather than on + // mount, to narrow the time-window and reduce false positives from other + // OTel collectors already ingesting data on the same cluster. + const [sessionStartTime, setSessionStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && sessionStartTime === null) { + setSessionStartTime(new Date().toISOString()); + } + }, [isMonitoringStepActive, sessionStartTime]); + const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ - isMonitoringActive: isMonitoringStepActive, - sessionStartTime, + isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', fetchInterval: FETCH_INTERVAL, troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, flowType: 'otel_logs', onboardingId: setupData?.onboardingId ?? '', endpoint: '/internal/observability_onboarding/otel_host/has-data', + extraQueryParams: { osType: OS_TYPE_MAP[selectedTab] }, }); const isMetricsOnboardingEnabled = usePricingFeature( @@ -149,31 +167,31 @@ export const OtelLogsPanel: React.FC = () => { () => [ ...(deeplinks?.logs ? [ - { - id: 'logs', - title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { - defaultMessage: 'View and analyze your logs', - }), - label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { - defaultMessage: 'Explore logs', - }), - href: deeplinks.logs, - }, - ] + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { + defaultMessage: 'View and analyze your logs', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: deeplinks.logs, + }, + ] : []), ...(isMetricsOnboardingEnabled && deeplinks?.metrics ? [ - { - id: 'metrics', - title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { - defaultMessage: 'View and analyze your metrics', - }), - label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { - defaultMessage: 'Open Hosts', - }), - href: deeplinks.metrics, - }, - ] + { + id: 'metrics', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { + defaultMessage: 'View and analyze your metrics', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { + defaultMessage: 'Open Hosts', + }), + href: deeplinks.metrics, + }, + ] : []), ], [deeplinks, isMetricsOnboardingEnabled] @@ -187,15 +205,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'linux', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'linux', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: 'sudo ./otelcol --config otel.yml', codeLanguage: 'sh', @@ -206,15 +224,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'mac', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'mac', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: './otelcol --config otel.yml', codeLanguage: 'sh', @@ -225,15 +243,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'windows', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'windows', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: '.\\otelcol.ps1 --config otel.yml', codeLanguage: 'powershell', @@ -242,8 +260,6 @@ export const OtelLogsPanel: React.FC = () => { [setupData, isMetricsOnboardingEnabled, isManagedOtlpServiceAvailable, useWiredStreams] ); - const [selectedTab, setSelectedTab] = React.useState(installTabContents[0].id); - const selectedContent = installTabContents.find((tab) => tab.id === selectedTab)!; if (error) { @@ -382,19 +398,19 @@ export const OtelLogsPanel: React.FC = () => {

{selectedTab === 'windows' ? i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.windowsLogDescription', - { - defaultMessage: - 'On Windows, logs are collected from the Windows Event Log. You can customize this in the otel.yml file.', - } - ) + 'xpack.observability_onboarding.otelLogsPanel.windowsLogDescription', + { + defaultMessage: + 'On Windows, logs are collected from the Windows Event Log. You can customize this in the otel.yml file.', + } + ) : i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2', - { - defaultMessage: - 'The default log path is /var/log/*. You can change this path in the otel.yml file if needed.', - } - )} + 'xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2', + { + defaultMessage: + 'The default log path is /var/log/*. You can change this path in the otel.yml file if needed.', + } + )}

@@ -422,21 +438,21 @@ export const OtelLogsPanel: React.FC = () => { status: (hasData ? 'complete' : isMonitoringStepActive - ? 'current' - : 'incomplete') as EuiStepStatus, + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> (null); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const [noDataPollCount, setNoDataPollCount] = useState(0); const { services: { analytics }, } = useKibana(); @@ -50,6 +57,13 @@ export function useTimeWindowDataDetection({ [JSON.stringify(extraQueryParams)] ); + // After FALLBACK_POLL_THRESHOLD consecutive "no data" responses, + // drop extra filters (e.g. osType) and fall back to basic time-window. + const hasExtraParams = + stableExtraQueryParams !== undefined && Object.keys(stableExtraQueryParams).length > 0; + const shouldFallback = hasExtraParams && noDataPollCount >= FALLBACK_POLL_THRESHOLD; + const effectiveExtraParams = shouldFallback ? undefined : stableExtraQueryParams; + const { data: hasDataResponse, status: hasDataStatus, @@ -59,11 +73,11 @@ export function useTimeWindowDataDetection({ if (!isMonitoringActive) return; return callApi(`GET ${endpoint}` as any, { params: { - query: { start: sessionStartTime, ...stableExtraQueryParams }, + query: { start: sessionStartTime, ...effectiveExtraParams }, }, }); }, - [isMonitoringActive, sessionStartTime, endpoint, stableExtraQueryParams], + [isMonitoringActive, sessionStartTime, endpoint, effectiveExtraParams], { showToastOnError: false } ); @@ -72,6 +86,9 @@ export function useTimeWindowDataDetection({ if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { return; } + if (hasDataResponse?.hasData === false) { + setNoDataPollCount((prev) => prev + 1); + } const timeout = setTimeout(() => { refetchHasData(); }, fetchInterval); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index 17c8a907e6ce7..e91ef06c38847 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -85,9 +85,7 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/otel_host/has-data', params: t.type({ - query: t.type({ - start: t.string, - }), + query: t.intersection([t.type({ start: t.string }), t.partial({ osType: t.string })]), }), security: { authz: { @@ -96,14 +94,18 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ }, }, async handler(resources): Promise<{ hasData: boolean }> { - const { start } = resources.params.query; + const { start, osType } = resources.params.query; const { elasticsearch } = await resources.context.core; try { + const filters: estypes.QueryDslQueryContainer[] = [ + { range: { '@timestamp': { gte: start } } }, + ]; + if (osType) { + filters.push({ term: { 'host.os.type': osType } }); + } const query: estypes.QueryDslQueryContainer = { - bool: { - filter: [{ range: { '@timestamp': { gte: start } } }], - }, + bool: { filter: filters }, }; const [logsResult, metricsResult] = await Promise.all([ From 926baddaf6ffb0a18e48ba5ef65f8942effeed3d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:44:46 +0000 Subject: [PATCH 10/21] Changes from node scripts/eslint_all_files --no-cache --fix --- .../quickstart_flows/otel_logs/index.tsx | 138 +++++++++--------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index f1b07bc1b99b2..89fb4a91435dc 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -167,31 +167,31 @@ export const OtelLogsPanel: React.FC = () => { () => [ ...(deeplinks?.logs ? [ - { - id: 'logs', - title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { - defaultMessage: 'View and analyze your logs', - }), - label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { - defaultMessage: 'Explore logs', - }), - href: deeplinks.logs, - }, - ] + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { + defaultMessage: 'View and analyze your logs', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: deeplinks.logs, + }, + ] : []), ...(isMetricsOnboardingEnabled && deeplinks?.metrics ? [ - { - id: 'metrics', - title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { - defaultMessage: 'View and analyze your metrics', - }), - label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { - defaultMessage: 'Open Hosts', - }), - href: deeplinks.metrics, - }, - ] + { + id: 'metrics', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { + defaultMessage: 'View and analyze your metrics', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { + defaultMessage: 'Open Hosts', + }), + href: deeplinks.metrics, + }, + ] : []), ], [deeplinks, isMetricsOnboardingEnabled] @@ -205,15 +205,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'linux', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'linux', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: 'sudo ./otelcol --config otel.yml', codeLanguage: 'sh', @@ -224,15 +224,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'mac', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'mac', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: './otelcol --config otel.yml', codeLanguage: 'sh', @@ -243,15 +243,15 @@ export const OtelLogsPanel: React.FC = () => { firstStepTitle: HOST_COMMAND, content: setupData ? buildInstallCommand({ - platform: 'windows', - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, - elasticsearchUrl: setupData.elasticsearchUrl, - apiKeyEncoded: setupData.apiKeyEncoded, - agentVersion: setupData.elasticAgentVersionInfo.agentVersion, - useWiredStreams, - }) + platform: 'windows', + isMetricsOnboardingEnabled, + isManagedOtlpServiceAvailable, + managedOtlpServiceUrl: setupData.managedOtlpServiceUrl, + elasticsearchUrl: setupData.elasticsearchUrl, + apiKeyEncoded: setupData.apiKeyEncoded, + agentVersion: setupData.elasticAgentVersionInfo.agentVersion, + useWiredStreams, + }) : '', start: '.\\otelcol.ps1 --config otel.yml', codeLanguage: 'powershell', @@ -398,19 +398,19 @@ export const OtelLogsPanel: React.FC = () => {

{selectedTab === 'windows' ? i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.windowsLogDescription', - { - defaultMessage: - 'On Windows, logs are collected from the Windows Event Log. You can customize this in the otel.yml file.', - } - ) + 'xpack.observability_onboarding.otelLogsPanel.windowsLogDescription', + { + defaultMessage: + 'On Windows, logs are collected from the Windows Event Log. You can customize this in the otel.yml file.', + } + ) : i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2', - { - defaultMessage: - 'The default log path is /var/log/*. You can change this path in the otel.yml file if needed.', - } - )} + 'xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2', + { + defaultMessage: + 'The default log path is /var/log/*. You can change this path in the otel.yml file if needed.', + } + )}

@@ -438,21 +438,21 @@ export const OtelLogsPanel: React.FC = () => { status: (hasData ? 'complete' : isMonitoringStepActive - ? 'current' - : 'incomplete') as EuiStepStatus, + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> Date: Fri, 20 Mar 2026 16:32:18 +0100 Subject: [PATCH 11/21] remove `any` --- .../shared/use_time_window_data_detection.ts | 4 ++-- .../lib/handle_has_data_search_error.ts | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts index 084fd162036d4..f13e6b3af6992 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts @@ -69,9 +69,9 @@ export function useTimeWindowDataDetection({ status: hasDataStatus, refetch: refetchHasData, } = useFetcher( - (callApi) => { + (callApi): Promise<{ hasData: boolean }> | undefined => { if (!isMonitoringActive) return; - return callApi(`GET ${endpoint}` as any, { + return callApi(`GET ${endpoint}` as Parameters[0], { params: { query: { start: sessionStartTime, ...effectiveExtraParams }, }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts index ac7abb2720c5e..5f91d015de628 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts @@ -6,15 +6,23 @@ */ import Boom from '@hapi/boom'; +import { errors } from '@elastic/elasticsearch'; /** * Handles errors from has-data Elasticsearch search requests. * Returns true if the error is a transient "no shards available" condition * (which should be treated as hasData: false), or throws a Boom error otherwise. */ -export function isNoShardsAvailableError(error: any): boolean { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; +export function isNoShardsAvailableError(error: unknown): boolean { + if (!(error instanceof errors.ResponseError)) { + return false; + } + + const body = error.body as + | { error?: { type?: string; root_cause?: Array<{ type?: string }> } } + | undefined; + const errorType = body?.error?.type; + const rootCauseType = body?.error?.root_cause?.[0]?.type; if ( errorType === 'search_phase_execution_exception' && @@ -26,6 +34,7 @@ export function isNoShardsAvailableError(error: any): boolean { return false; } -export function throwHasDataSearchError(error: any): never { - throw Boom.internal(`Elasticsearch responded with an error. ${error.message}`); +export function throwHasDataSearchError(error: unknown): never { + const message = error instanceof Error ? error.message : String(error); + throw Boom.internal(`Elasticsearch responded with an error. ${message}`); } From d7f978aef2b83f8cc480a1503a0beed0b3627a70 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Mon, 23 Mar 2026 16:09:03 +0100 Subject: [PATCH 12/21] address comments, resolve CI problems --- .../translations/translations/de-DE.json | 8 -- .../translations/translations/fr-FR.json | 8 -- .../translations/translations/ja-JP.json | 8 -- .../translations/translations/zh-CN.json | 8 -- .../stateful/pom/pages/otel_host_flow.page.ts | 4 +- .../pom/pages/otel_kubernetes_flow.page.ts | 4 +- .../kubernetes/data_ingest_status.tsx | 6 +- .../quickstart_flows/otel_logs/index.tsx | 8 -- .../server/routes/kubernetes/route.ts | 16 +++- .../server/routes/otel_host/route.ts | 74 +++++++++---------- 10 files changed, 58 insertions(+), 86 deletions(-) diff --git a/x-pack/platform/plugins/private/translations/translations/de-DE.json b/x-pack/platform/plugins/private/translations/translations/de-DE.json index 6d7f31fa9eef8..1621d495f28e8 100644 --- a/x-pack/platform/plugins/private/translations/translations/de-DE.json +++ b/x-pack/platform/plugins/private/translations/translations/de-DE.json @@ -29202,7 +29202,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumentieren Sie Ihre Anwendung (optional)", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Überprüfen Sie die Gesundheit Ihres Kubernetes-Clusters:", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Daten visualisieren", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysieren Sie die Gesundheit Ihres Kubernetes-Clusters und überwachen Sie Ihre Container-Workloads.", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "Weitere Informationen finden Sie in der Dokumentation", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Wählen Sie eine Programmiersprache aus.", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Erkunden Sie das Service-Inventar", @@ -29213,9 +29212,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "Fehler bei der Installation der Integration", "xpack.observability_onboarding.otelLogs.status.failedDetails": "Eingehende Daten könnten möglicherweise nicht korrekt indiziert werden. Details:", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Plattform wählen", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Dokumentation öffnen", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Logs durchsuchen", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Offene Hosts", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Ab dem Setup werden neue log Meldungen gesammelt.", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Der Standardlogpfad ist /var/log/*. Sie können diesen Pfad bei Bedarf in der Datei otel.yml ändern.", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informationen zur Konfiguration", @@ -29226,10 +29222,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Daten visualisieren", "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.label": "Technische Vorschau", "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.tooltip": "Diese Funktionalität befindet sich in der technischen Vorschau und kann in einer zukünftigen Version geändert oder vollständig entfernt werden. Elastic wird sich bemühen, alle Probleme zu beheben, aber die Features in der technischen Vorschau unterliegen nicht dem Support-SLA der offiziellen GA-Features.", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Weitere Einzelheiten und Lösungen zur Fehlerbehebung finden Sie in unserer Dokumentation. {link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Ihre Metriken anzeigen und analysieren", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Ihre Logs ansehen und analysieren", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Kehren Sie nach der Ausführung des vorherigen Befehls zurück und sehen Sie sich Ihre Daten an.", "xpack.observability_onboarding.otelTile.description": "Erfassen Sie Protokolle und Host-Metriken mit der Elastic-Distribution des OpenTelemetry Collector", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "Laden Sie Daten aus einer CSV-, TSV-, JSON- oder anderen Log-Datei zur Analyse in Elasticsearch hoch.", diff --git a/x-pack/platform/plugins/private/translations/translations/fr-FR.json b/x-pack/platform/plugins/private/translations/translations/fr-FR.json index 9b8a79ee7a331..f8e2479f8ae00 100644 --- a/x-pack/platform/plugins/private/translations/translations/fr-FR.json +++ b/x-pack/platform/plugins/private/translations/translations/fr-FR.json @@ -29550,7 +29550,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumenter votre application (facultatif)", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Vérifiez l'intégrité de votre cluster Kubernetes :", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Visualiser vos données", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysez l'intégrité de votre cluster Kubernetes et monitorez les charges de travail de vos conteneurs.", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "se reporter à la documentation", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Sélectionner un langage de programmation", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Explorer l'inventaire des services", @@ -29561,9 +29560,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "Échec de l'installation de l'intégration", "xpack.observability_onboarding.otelLogs.status.failedDetails": "Les données entrantes peuvent ne pas être indexées correctement. Détails :", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Choisissez une plateforme", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Ouvrir la documentation", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Explorer les logs", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Ouvrir les hôtes", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Les nouveaux messages de log sont collectés à partir de la configuration.", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Le chemin des logs par défaut est /var/log/*. Vous pouvez si nécessaire modifier ce chemin dans le fichier otel.yml.", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informations sur la configuration", @@ -29572,10 +29568,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "Sélectionnez votre plateforme", "xpack.observability_onboarding.otelLogsPanel.steps.start": "Lancez le collecteur", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Visualiser vos données", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Vous trouverez plus d'informations et une solution de résolution des problèmes dans notre documentation. {link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Visualisez et analysez vos métriques", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Visualisez et analysez vos logs", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Après avoir exécuté la commande précédente, revenez et visualisez vos données.", "xpack.observability_onboarding.otelTile.description": "Collectez les logs et les indicateurs de l'hôte à l'aide de la distribution Elastic du collecteur OpenTelemetry", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "Téléchargez les données d'un fichier CSV, TSV, JSON ou autre fichier log vers Elasticsearch pour analyse.", diff --git a/x-pack/platform/plugins/private/translations/translations/ja-JP.json b/x-pack/platform/plugins/private/translations/translations/ja-JP.json index e2a1fb360e1d6..73a6cf75edc39 100644 --- a/x-pack/platform/plugins/private/translations/translations/ja-JP.json +++ b/x-pack/platform/plugins/private/translations/translations/ja-JP.json @@ -29604,7 +29604,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "アプリケーションのインストルメンテーション（任意）", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Kubernetesクラスター正常性を確認：", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "データを可視化する", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Kubernetesクラスターの正常性を分析し、コンテナーのワークロードを監視します。", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "ドキュメントを参照", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "プログラミング言語を選択", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "サービスインベントリを探索", @@ -29615,9 +29614,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "統合のインストールに失敗しました", "xpack.observability_onboarding.otelLogs.status.failedDetails": "受信データは正しくインデックス化されていない可能性があります。詳細：", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "プラットフォームを選択", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "ドキュメントを開く", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "ログを探索", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "ホストを開く", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "今後、新しいログメッセージはセットアップから収集されます。", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "デフォルトのログのパスは/var/log/*です。必要に応じて、otel.ymlファイルでこのパスを変更できます。", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "構成情報", @@ -29626,10 +29622,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "プラットフォームを選択", "xpack.observability_onboarding.otelLogsPanel.steps.start": "コレクターを開始", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "データを可視化する", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "詳細とトラブルシューティングの解決策については、ドキュメントをご覧ください。{link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "メトリックを表示して分析", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "ログを表示して分析", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "前のコマンドを実行した後、データに戻って表示します。", "xpack.observability_onboarding.otelTile.description": "OpenTelemetryコレクターのElasticディストリビューションを使用して、ログとホストメトリックを収集します。", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "分析するため、CSV、TSV、JSON、他のログファイルからElasticsearchにアップロードします。", diff --git a/x-pack/platform/plugins/private/translations/translations/zh-CN.json b/x-pack/platform/plugins/private/translations/translations/zh-CN.json index a59f2fd59e432..5e97153670e04 100644 --- a/x-pack/platform/plugins/private/translations/translations/zh-CN.json +++ b/x-pack/platform/plugins/private/translations/translations/zh-CN.json @@ -29583,7 +29583,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "检测您的应用程序（可选）", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "检查 Kubernetes 集群的运行状况：", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "可视化数据", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "分析 Kubernetes 集群的运行状况并监测容器工作负载。", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "参阅文档", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "选择编程语言", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "浏览服务库存", @@ -29594,9 +29593,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "集成安装失败", "xpack.observability_onboarding.otelLogs.status.failedDetails": "传入数据可能未正确索引。详情：", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "选择平台", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "打开文档", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "浏览日志", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "打开主机", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "将从设置完成后收集新的日志消息。", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "默认日志路径为 /var/log/*。如果需要，可以在 otel.yml 文件中更改此路径。", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "配置信息", @@ -29605,10 +29601,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "选择平台", "xpack.observability_onboarding.otelLogsPanel.steps.start": "启动收集器", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "可视化数据", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "在我们的文档中查找更多详情和故障排除解决方案。{link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "查看并分析您的指标", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "查看并分析您的日志", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "运行上一个命令后，返回并查看您的数据。", "xpack.observability_onboarding.otelTile.description": "使用 OpenTelemetry 收集器的 Elastic 发行版收集日志和主机指标", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "从 CSV、TSV、JSON 或其他日志文件上传数据到 Elasticsearch 以进行分析。", diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts index 27d64a4e2143f..84704894353c1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts @@ -64,11 +64,11 @@ export class OtelHostFlowPage { await this.exploreLogsButton.click(); } - public async assertDataReceivedIndicator() { + public async assertDataReceivedIndicator(): Promise { await expect( this.dataReceivedIndicator, 'Data received indicator should be visible' - ).toBeVisible(); + ).toBeVisible({ timeout: 5 * 60_000 }); } public async assertLogsExplorationButtonVisible() { diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts index 689d1ae2be437..1b604614da30e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts @@ -86,11 +86,11 @@ export class OtelKubernetesFlowPage { } } - public async assertDataReceivedIndicator() { + public async assertDataReceivedIndicator(): Promise { await expect( this.dataReceivedIndicator, 'Data received indicator should be visible' - ).toBeVisible(); + ).toBeVisible({ timeout: 5 * 60_000 }); } public async assertLogsExplorationButtonVisible() { diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index e8032045cac22..4e9fd7beb0f4c 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -41,6 +41,7 @@ export function DataIngestStatus({ }: Props) { const [checkDataStartTime] = useState(Date.now()); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const [dataReceivedNotified, setDataReceivedNotified] = useState(false); const { services: { analytics }, } = useKibana(); @@ -91,10 +92,11 @@ export function DataIngestStatus({ // This drives the step status to 'complete' and must wait for metrics // if any action link requires them. useEffect(() => { - if (isReady) { + if (isReady && !dataReceivedNotified) { onDataReceived?.(); + setDataReceivedNotified(true); } - }, [isReady, onDataReceived]); + }, [isReady, onDataReceived, dataReceivedNotified]); const isTroubleshootingVisible = hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index 89fb4a91435dc..f963fe668ee8d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -62,13 +62,6 @@ const HOST_COMMAND = i18n.translate( const FETCH_INTERVAL = 2000; const SHOW_TROUBLESHOOTING_DELAY = 120_000; -// Used to scope time-window detection to the user's OS and avoid false positives from other hosts. -const OS_TYPE_MAP: Record = { - linux: 'linux', - mac: 'darwin', - windows: 'windows', -}; - export const OtelLogsPanel: React.FC = () => { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.autoDetectPanel.breadcrumbs.otelHost', { @@ -128,7 +121,6 @@ export const OtelLogsPanel: React.FC = () => { flowType: 'otel_logs', onboardingId: setupData?.onboardingId ?? '', endpoint: '/internal/observability_onboarding/otel_host/has-data', - extraQueryParams: { osType: OS_TYPE_MAP[selectedTab] }, }); const isMetricsOnboardingEnabled = usePricingFeature( diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index 4b012f46dcb95..6122f951b6a3e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -171,7 +171,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; - const [logsResult, metricsResult] = await Promise.all([ + const [logsResult, metricsResult] = await Promise.allSettled([ elasticsearch.client.asCurrentUser.search({ index: ['logs-*', 'logs.*'], ignore_unavailable: true, @@ -192,8 +192,18 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }), ]); - const hasLogs = (logsResult.hits.total as estypes.SearchTotalHits).value > 0; - const hasMetrics = (metricsResult.hits.total as estypes.SearchTotalHits).value > 0; + const resolveProbe = (result: PromiseSettledResult): boolean => { + if (result.status === 'fulfilled') { + return (result.value.hits.total as estypes.SearchTotalHits).value > 0; + } + if (isNoShardsAvailableError(result.reason)) { + return false; + } + throwHasDataSearchError(result.reason); + }; + + const hasLogs = resolveProbe(logsResult); + const hasMetrics = resolveProbe(metricsResult); return { hasData: hasLogs || hasMetrics, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index e91ef06c38847..49b497617a05c 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -97,47 +97,47 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ const { start, osType } = resources.params.query; const { elasticsearch } = await resources.context.core; - try { - const filters: estypes.QueryDslQueryContainer[] = [ - { range: { '@timestamp': { gte: start } } }, - ]; - if (osType) { - filters.push({ term: { 'host.os.type': osType } }); - } - const query: estypes.QueryDslQueryContainer = { - bool: { filter: filters }, - }; - - const [logsResult, metricsResult] = await Promise.all([ - elasticsearch.client.asCurrentUser.search({ - index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query, - }), - elasticsearch.client.asCurrentUser.search({ - index: ['metrics-*.otel-*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query, - }), - ]); + const filters: estypes.QueryDslQueryContainer[] = [{ range: { '@timestamp': { gte: start } } }]; + if (osType) { + filters.push({ term: { 'host.os.type': osType } }); + } + const query: estypes.QueryDslQueryContainer = { + bool: { filter: filters }, + }; - const hasLogs = (logsResult.hits.total as estypes.SearchTotalHits).value > 0; - const hasMetrics = (metricsResult.hits.total as estypes.SearchTotalHits).value > 0; + const [logsResult, metricsResult] = await Promise.allSettled([ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*.otel-*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]); - return { hasData: hasLogs || hasMetrics }; - } catch (error) { - if (isNoShardsAvailableError(error)) { - return { hasData: false }; + const resolveProbe = (result: PromiseSettledResult): boolean => { + if (result.status === 'fulfilled') { + return (result.value.hits.total as estypes.SearchTotalHits).value > 0; } + if (isNoShardsAvailableError(result.reason)) { + return false; + } + throwHasDataSearchError(result.reason); + }; - throwHasDataSearchError(error); - } + const hasLogs = resolveProbe(logsResult); + const hasMetrics = resolveProbe(metricsResult); + + return { hasData: hasLogs || hasMetrics }; }, }); From 40abee36b2f7c01fa6dfd1f542cd734fe55c549a Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Mon, 23 Mar 2026 20:21:42 +0100 Subject: [PATCH 13/21] new approach for otel apm --- .../quickstart_flows/otel_apm/index.tsx | 37 +++++++++++++++++-- .../server/routes/otel_apm/route.ts | 19 +++++----- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index a23d61469c32b..1213faef688ff 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -14,6 +14,8 @@ import type { EuiStepStatus } from '@elastic/eui'; import { EuiBasicTable, EuiButtonIcon, + EuiFieldText, + EuiFormRow, EuiLink, EuiMarkdownFormat, EuiPanel, @@ -53,6 +55,7 @@ export function OtelApmQuickstartFlow() { } = useKibana(); const { data, status, error, refetch } = useOtelApmFlow(); const { onPageReady } = usePerformanceContext(); + const [serviceName, setServiceName] = useState(''); useEffect(() => { if (data) { @@ -80,6 +83,7 @@ export function OtelApmQuickstartFlow() { } }, [isMonitoringStepActive, sessionStartTime]); + const trimmedServiceName = serviceName.trim(); const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, sessionStartTime: sessionStartTime ?? '', @@ -88,6 +92,7 @@ export function OtelApmQuickstartFlow() { flowType: 'otel_apm', onboardingId: data?.onboardingId ?? '', endpoint: '/internal/observability_onboarding/otel_apm/has-data', + extraQueryParams: trimmedServiceName ? { serviceName: trimmedServiceName } : undefined, }); if (error !== undefined) { @@ -131,6 +136,8 @@ export function OtelApmQuickstartFlow() { )} @@ -285,10 +292,15 @@ function InstallSDKInstructions() { function ConfigureSDKInstructions({ managedOtlpServiceUrl, apiKeyEncoded, + serviceName, + onServiceNameChange, }: { managedOtlpServiceUrl: string; apiKeyEncoded: string; + serviceName: string; + onServiceNameChange: (value: string) => void; }) { + const serviceNameDisplay = serviceName.trim() || ''; const items = [ { setting: 'OTEL_EXPORTER_OTLP_ENDPOINT', @@ -300,8 +312,7 @@ function ConfigureSDKInstructions({ }, { setting: 'OTEL_RESOURCE_ATTRIBUTES', - value: - 'service.name=,service.version=,deployment.environment=production', + value: `service.name=${serviceNameDisplay},service.version=,deployment.environment=production`, }, ]; @@ -342,10 +353,30 @@ function ConfigureSDKInstructions({ return ( <> + + onServiceNameChange(e.target.value)} + /> + + {i18n.translate('xpack.observability_onboarding.otelApm.configureAgent.textPre', { defaultMessage: - 'Set the following variables in your application’s environment to configure the SDK:', + "Set the following variables in your application's environment to configure the SDK:", })} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts index f279b9712fdf3..2580ce58693f1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts @@ -60,9 +60,7 @@ const createOtelApmOnboardingFlowRoute = createObservabilityOnboardingServerRout const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/otel_apm/has-data', params: t.type({ - query: t.type({ - start: t.string, - }), + query: t.intersection([t.type({ start: t.string }), t.partial({ serviceName: t.string })]), }), security: { authz: { @@ -71,19 +69,20 @@ const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ }, }, async handler(resources): Promise<{ hasData: boolean }> { - const { start } = resources.params.query; + const { start, serviceName } = resources.params.query; const { elasticsearch } = await resources.context.core; try { + const filters: estypes.QueryDslQueryContainer[] = [ + { range: { '@timestamp': { gte: start } } }, + ]; + if (serviceName) { + filters.push({ term: { 'service.name': serviceName } }); + } const query: estypes.QueryDslQueryContainer = { - bool: { - filter: [{ range: { '@timestamp': { gte: start } } }], - }, + bool: { filter: filters }, }; - // Time-window detection: matches any APM data arriving after session start. - // May produce false positives if other services are already active. - // Correlation ID fallback (labels.onboarding_id) is available if needed. const result = await elasticsearch.client.asCurrentUser.search({ index: [ 'traces-apm*', From 80796860fa83412a8345203bafe63e111153b9ff Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Mon, 30 Mar 2026 17:10:18 +0200 Subject: [PATCH 14/21] fix K8s has-data timeout on serverless by scoping runtime mapping to wired stream indices --- .../server/routes/kubernetes/route.ts | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index 6122f951b6a3e..68be6fe67ebd6 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -138,7 +138,15 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ const { elasticsearch } = await resources.context.core; try { - const query: estypes.QueryDslQueryContainer = { + const commonSearchParams = { + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0 as const, + terminate_after: 1, + }; + + // Indexed fields only, no runtime mapping (broad logs-*/metrics-* would time out with scripts). + const indexedQuery: estypes.QueryDslQueryContainer = { bool: { filter: [ { @@ -146,7 +154,6 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ should: [ ...termQuery('fields.onboarding_id', onboardingId), ...termQuery('resource.attributes.onboarding.id', onboardingId), - ...termQuery('resource.attributes.onboarding.id._rt', onboardingId), ...termQuery('labels.onboarding_id', onboardingId), ], minimum_should_match: 1, @@ -156,11 +163,12 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; - // Logs Essentials + Wired Streams: logs.otel uses a passthrough mapping for - // resource.attributes, storing fields in _source without indexing them. - // We use a distinct runtime field name so it does not shadow the indexed - // mapping on classic streams where resource.attributes.onboarding.id is - // already a keyword. The query includes both names in the should clause. + // Logs Essentials + Wired Streams: logs.otel uses a passthrough mapping + // for resource.attributes, storing fields in _source without indexing + // them. A runtime field extracts onboarding.id at query time. + // We scope this to wired stream indices only (logs.*, metrics.*) to + // avoid running the script across all classic data streams which would + // time out on large clusters. const runtimeMappings: estypes.MappingRuntimeFields = { 'resource.attributes.onboarding.id._rt': { type: 'keyword', @@ -171,26 +179,39 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; - const [logsResult, metricsResult] = await Promise.allSettled([ - elasticsearch.client.asCurrentUser.search({ - index: ['logs-*', 'logs.*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - runtime_mappings: runtimeMappings, - query, - }), - elasticsearch.client.asCurrentUser.search({ - index: ['metrics-*', 'metrics.*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - runtime_mappings: runtimeMappings, - query, - }), - ]); + const wiredStreamQuery: estypes.QueryDslQueryContainer = { + bool: { + filter: termQuery('resource.attributes.onboarding.id._rt', onboardingId), + }, + }; + + const [logsResult, metricsResult, wiredLogsResult, wiredMetricsResult] = + await Promise.allSettled([ + // Fast: indexed fields on broad index patterns, no runtime mapping + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*'], + ...commonSearchParams, + query: indexedQuery, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*'], + ...commonSearchParams, + query: indexedQuery, + }), + // Scoped: runtime mapping only on wired stream indices + elasticsearch.client.asCurrentUser.search({ + index: ['logs.*'], + ...commonSearchParams, + runtime_mappings: runtimeMappings, + query: wiredStreamQuery, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics.*'], + ...commonSearchParams, + runtime_mappings: runtimeMappings, + query: wiredStreamQuery, + }), + ]); const resolveProbe = (result: PromiseSettledResult): boolean => { if (result.status === 'fulfilled') { @@ -202,8 +223,8 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ throwHasDataSearchError(result.reason); }; - const hasLogs = resolveProbe(logsResult); - const hasMetrics = resolveProbe(metricsResult); + const hasLogs = resolveProbe(logsResult) || resolveProbe(wiredLogsResult); + const hasMetrics = resolveProbe(metricsResult) || resolveProbe(wiredMetricsResult); return { hasData: hasLogs || hasMetrics, From 6c73c8d89f516bf96355650a9cf0c2e91d73f5c8 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 31 Mar 2026 11:49:10 +0200 Subject: [PATCH 15/21] fix K8s has-data timeout on serverless by dropping runtime mapping for wired streams Replace the Painless runtime mapping (which timed out on clusters with real data volume) with a time-range-only fallback for wired stream indices. Classic data streams still use fast indexed onboarding ID fields. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../kubernetes/data_ingest_status.tsx | 6 +- .../resolve_has_data_probes.test.ts | 82 ++++++++++++++++ .../kubernetes/resolve_has_data_probes.ts | 26 +++++ .../server/routes/kubernetes/route.ts | 97 ++++++++----------- 4 files changed, 154 insertions(+), 57 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index 5e7dac1af4456..93e533df8c69c 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -46,13 +46,15 @@ export function DataIngestStatus({ services: { analytics }, } = useKibana(); + const startIso = new Date(checkDataStartTime).toISOString(); + const { data, status, refetch } = useFetcher( (callApi) => { return callApi('GET /internal/observability_onboarding/kubernetes/{onboardingId}/has-data', { - params: { path: { onboardingId } }, + params: { path: { onboardingId }, query: { start: startIso } }, }); }, - [onboardingId] + [onboardingId, startIso] ); const hasData = data?.hasData ?? false; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts new file mode 100644 index 0000000000000..e3c60986e4c52 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import type { estypes } from '@elastic/elasticsearch'; +import { resolveProbe } from './resolve_has_data_probes'; + +const fulfilledWithHits = (count: number): PromiseSettledResult => ({ + status: 'fulfilled', + value: { + hits: { total: { value: count, relation: 'eq' }, max_score: null, hits: [] }, + } as unknown as estypes.SearchResponse, +}); + +const rejectedWith = (error: Error): PromiseSettledResult => ({ + status: 'rejected', + reason: error, +}); + +const noShardsError = () => { + const error = new errors.ResponseError({ + statusCode: 503, + body: { + error: { + type: 'search_phase_execution_exception', + root_cause: [{ type: 'no_shard_available_action_exception' }], + }, + }, + headers: {}, + warnings: [], + meta: {} as never, + }); + return error; +}; + +describe('resolveProbe', () => { + it('returns true when documents are found', () => { + expect(resolveProbe(fulfilledWithHits(1))).toBe(true); + }); + + it('returns false when no documents are found', () => { + expect(resolveProbe(fulfilledWithHits(0))).toBe(false); + }); + + it('returns false on no shards available error', () => { + expect(resolveProbe(rejectedWith(noShardsError()))).toBe(false); + }); + + it('throws on unexpected errors', () => { + expect(() => resolveProbe(rejectedWith(new Error('Request timed out')))).toThrow( + 'Elasticsearch responded with an error. Request timed out' + ); + }); + + describe('combined classic + wired stream probes', () => { + it('returns true when classic probe finds data and wired probe finds nothing', () => { + const hasLogs = resolveProbe(fulfilledWithHits(1)) || resolveProbe(fulfilledWithHits(0)); + expect(hasLogs).toBe(true); + }); + + it('returns true when classic probe finds nothing but wired probe finds data', () => { + const hasLogs = resolveProbe(fulfilledWithHits(0)) || resolveProbe(fulfilledWithHits(1)); + expect(hasLogs).toBe(true); + }); + + it('returns false when both probes find nothing', () => { + const hasLogs = resolveProbe(fulfilledWithHits(0)) || resolveProbe(fulfilledWithHits(0)); + expect(hasLogs).toBe(false); + }); + + it('handles optional wired probe (undefined when no start time)', () => { + const wiredResult: PromiseSettledResult | undefined = undefined; + const hasLogs = + resolveProbe(fulfilledWithHits(0)) || (wiredResult ? resolveProbe(wiredResult) : false); + expect(hasLogs).toBe(false); + }); + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts new file mode 100644 index 0000000000000..bfaef76352799 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; + +/** + * Resolves a has-data probe result. Returns true if documents were found, + * false for no-shards-available errors, and throws on unexpected errors. + */ +export const resolveProbe = (result: PromiseSettledResult): boolean => { + if (result.status === 'fulfilled') { + return (result.value.hits.total as estypes.SearchTotalHits).value > 0; + } + if (isNoShardsAvailableError(result.reason)) { + return false; + } + throwHasDataSearchError(result.reason); +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index 68be6fe67ebd6..2af0ecebe9190 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -14,6 +14,7 @@ import { isNoShardsAvailableError, throwHasDataSearchError, } from '../../lib/handle_has_data_search_error'; +import { resolveProbe } from './resolve_has_data_probes'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; @@ -126,6 +127,9 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ path: t.type({ onboardingId: t.string, }), + query: t.partial({ + start: t.string, + }), }), security: { authz: { @@ -135,6 +139,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, async handler(resources): Promise { const { onboardingId } = resources.params.path; + const { start } = resources.params.query; const { elasticsearch } = await resources.context.core; try { @@ -145,7 +150,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ terminate_after: 1, }; - // Indexed fields only, no runtime mapping (broad logs-*/metrics-* would time out with scripts). + // Classic data streams: use indexed onboarding ID fields (fast inverted-index lookups). const indexedQuery: estypes.QueryDslQueryContainer = { bool: { filter: [ @@ -163,68 +168,50 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; - // Logs Essentials + Wired Streams: logs.otel uses a passthrough mapping - // for resource.attributes, storing fields in _source without indexing - // them. A runtime field extracts onboarding.id at query time. - // We scope this to wired stream indices only (logs.*, metrics.*) to - // avoid running the script across all classic data streams which would - // time out on large clusters. - const runtimeMappings: estypes.MappingRuntimeFields = { - 'resource.attributes.onboarding.id._rt': { - type: 'keyword', - script: { - source: - "def v = params._source?.resource?.attributes?.get('onboarding.id'); if (v != null) emit(v.toString())", - }, - }, - }; - - const wiredStreamQuery: estypes.QueryDslQueryContainer = { - bool: { - filter: termQuery('resource.attributes.onboarding.id._rt', onboardingId), - }, - }; - - const [logsResult, metricsResult, wiredLogsResult, wiredMetricsResult] = - await Promise.allSettled([ - // Fast: indexed fields on broad index patterns, no runtime mapping - elasticsearch.client.asCurrentUser.search({ - index: ['logs-*'], - ...commonSearchParams, - query: indexedQuery, - }), - elasticsearch.client.asCurrentUser.search({ - index: ['metrics-*'], - ...commonSearchParams, - query: indexedQuery, - }), - // Scoped: runtime mapping only on wired stream indices + // Wired streams (logs.otel*, logs.ecs*) use passthrough mapping where + // onboarding.id is not indexed, so we cannot filter by it without a + // runtime mapping (which times out on large clusters). Instead, fall + // back to a time-range-only query when a start time is provided. + const wiredStreamQuery: estypes.QueryDslQueryContainer | undefined = start + ? { bool: { filter: [{ range: { '@timestamp': { gte: start } } }] } } + : undefined; + + const searches: Array> = [ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*'], + ...commonSearchParams, + query: indexedQuery, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*'], + ...commonSearchParams, + query: indexedQuery, + }), + ]; + + if (wiredStreamQuery) { + searches.push( elasticsearch.client.asCurrentUser.search({ - index: ['logs.*'], + index: ['logs.otel*', 'logs.ecs*'], ...commonSearchParams, - runtime_mappings: runtimeMappings, query: wiredStreamQuery, }), elasticsearch.client.asCurrentUser.search({ - index: ['metrics.*'], + index: ['metrics.otel*', 'metrics.ecs*'], ...commonSearchParams, - runtime_mappings: runtimeMappings, query: wiredStreamQuery, - }), - ]); - - const resolveProbe = (result: PromiseSettledResult): boolean => { - if (result.status === 'fulfilled') { - return (result.value.hits.total as estypes.SearchTotalHits).value > 0; - } - if (isNoShardsAvailableError(result.reason)) { - return false; - } - throwHasDataSearchError(result.reason); - }; + }) + ); + } + + const results = await Promise.allSettled(searches); + const [logsResult, metricsResult, wiredLogsResult, wiredMetricsResult] = results; - const hasLogs = resolveProbe(logsResult) || resolveProbe(wiredLogsResult); - const hasMetrics = resolveProbe(metricsResult) || resolveProbe(wiredMetricsResult); + const hasLogs = + resolveProbe(logsResult) || (wiredLogsResult ? resolveProbe(wiredLogsResult) : false); + const hasMetrics = + resolveProbe(metricsResult) || + (wiredMetricsResult ? resolveProbe(wiredMetricsResult) : false); return { hasData: hasLogs || hasMetrics, From 9ec039bdf27a37ac7f0435c159c7afefdaead8cf Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 31 Mar 2026 14:26:38 +0200 Subject: [PATCH 16/21] detect already flowing data, disable detection if true --- .../quickstart_flows/cloudforwarder/index.tsx | 46 ++++++----- .../kubernetes/data_ingest_status.tsx | 49 ++++++----- .../quickstart_flows/otel_logs/index.tsx | 46 ++++++----- .../shared/use_time_window_data_detection.ts | 17 +++- .../lib/check_pre_existing_data.test.ts | 82 +++++++++++++++++++ .../server/lib/check_pre_existing_data.ts | 46 +++++++++++ .../server/routes/cloudforwarder/route.ts | 30 ++++--- .../server/routes/kubernetes/route.ts | 22 ++++- .../server/routes/otel_host/route.ts | 47 ++++++----- 9 files changed, 280 insertions(+), 105 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx index 5858ef61737b9..6da361480f869 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx @@ -81,7 +81,7 @@ export function CloudForwarderPanel() { onboardingId: data?.onboardingId, }); - const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ isMonitoringActive: isMonitoringStepActive && monitoringLogType !== null && sessionStartTime !== null, sessionStartTime: sessionStartTime ?? '', @@ -359,32 +359,34 @@ export function CloudForwarderPanel() { defaultMessage: 'Visualize your data', } ), - status: (hasData + status: (hasData || hasPreExistingData ? 'complete' : isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> - + {!(hasPreExistingData && !hasData) && ( + + )} {isTroubleshootingVisible && ( <> @@ -413,7 +415,7 @@ export function CloudForwarderPanel() { )} - {hasData === true && ( + {(hasData === true || hasPreExistingData) && ( <> actionLink.requires === 'metrics'); const isReady = needsMetrics ? hasMetrics : hasData; @@ -67,7 +68,7 @@ export function DataIngestStatus({ useEffect(() => { const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(status) || isReady) { + if (pendingStatusList.includes(status) || isReady || hasPreExistingData) { return; } @@ -76,7 +77,7 @@ export function DataIngestStatus({ }, FETCH_INTERVAL); return () => clearTimeout(timeout); - }, [isReady, refetch, status]); + }, [isReady, hasPreExistingData, refetch, status]); useEffect(() => { if (hasData === true && !dataReceivedTelemetrySent) { @@ -103,19 +104,21 @@ export function DataIngestStatus({ const isTroubleshootingVisible = hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; - const filteredActionLinks = actionLinks.filter((actionLink) => { - const requires = actionLink.requires ?? 'any'; + const filteredActionLinks = hasPreExistingData + ? actionLinks + : actionLinks.filter((actionLink) => { + const requires = actionLink.requires ?? 'any'; - if (requires === 'logs') { - return hasLogs; - } + if (requires === 'logs') { + return hasLogs; + } - if (requires === 'metrics') { - return hasMetrics; - } + if (requires === 'metrics') { + return hasMetrics; + } - return hasData; - }); + return hasData; + }); const filteredActionLinksWithHref = filteredActionLinks.filter((actionLink) => Boolean(actionLink.href) @@ -145,15 +148,17 @@ export function DataIngestStatus({ return ( <> - + {!(hasPreExistingData && !hasData) && ( + + )} {isTroubleshootingVisible && ( <> @@ -184,7 +189,7 @@ export function DataIngestStatus({ )} - {hasData === true && filteredActionLinksWithHref.length > 0 && ( + {(hasData === true || hasPreExistingData) && filteredActionLinksWithHref.length > 0 && ( <> diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index 04a03c52cb6c3..f6fea1eeadd89 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -113,7 +113,7 @@ export const OtelLogsPanel: React.FC = () => { } }, [isMonitoringStepActive, sessionStartTime]); - const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, sessionStartTime: sessionStartTime ?? '', fetchInterval: FETCH_INTERVAL, @@ -427,32 +427,34 @@ export const OtelLogsPanel: React.FC = () => { defaultMessage: 'Visualize your data', } ), - status: (hasData + status: (hasData || hasPreExistingData ? 'complete' : isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> - + {!(hasPreExistingData && !hasData) && ( + + )} {isTroubleshootingVisible && ( <> @@ -481,7 +483,7 @@ export const OtelLogsPanel: React.FC = () => { )} - {hasData === true && visualizeActionLinks.length > 0 && ( + {(hasData === true || hasPreExistingData) && visualizeActionLinks.length > 0 && ( <> | undefined => { + (callApi): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> | undefined => { if (!isMonitoringActive) return; return callApi(`GET ${endpoint}` as Parameters[0], { params: { @@ -83,7 +83,11 @@ export function useTimeWindowDataDetection({ useEffect(() => { const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(hasDataStatus) || hasDataResponse?.hasData === true) { + if ( + pendingStatusList.includes(hasDataStatus) || + hasDataResponse?.hasData === true || + hasDataResponse?.hasPreExistingData === true + ) { return; } if (hasDataResponse?.hasData === false) { @@ -93,7 +97,13 @@ export function useTimeWindowDataDetection({ refetchHasData(); }, fetchInterval); return () => clearTimeout(timeout); - }, [hasDataResponse?.hasData, refetchHasData, hasDataStatus, fetchInterval]); + }, [ + hasDataResponse?.hasData, + hasDataResponse?.hasPreExistingData, + refetchHasData, + hasDataStatus, + fetchInterval, + ]); useEffect(() => { if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { @@ -122,6 +132,7 @@ export function useTimeWindowDataDetection({ return { hasData: hasDataResponse?.hasData ?? false, + hasPreExistingData: hasDataResponse?.hasPreExistingData ?? false, isTroubleshootingVisible, }; } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts new file mode 100644 index 0000000000000..4b1ac081fe42e --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import { checkPreExistingData } from './check_pre_existing_data'; + +const createMockEsClient = (response?: unknown, error?: Error) => { + const search = error ? jest.fn().mockRejectedValue(error) : jest.fn().mockResolvedValue(response); + return { search } as unknown as Parameters[0]; +}; + +const hitsResponse = (count: number) => ({ + hits: { total: { value: count, relation: 'eq' }, max_score: null, hits: [] }, +}); + +describe('checkPreExistingData', () => { + const indices = ['logs.otel*', 'metrics.otel*']; + const start = '2026-03-31T10:00:00.000Z'; + + it('returns true when documents exist before start', async () => { + const client = createMockEsClient(hitsResponse(5)); + expect(await checkPreExistingData(client, indices, start)).toBe(true); + }); + + it('returns false when no documents exist before start', async () => { + const client = createMockEsClient(hitsResponse(0)); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); + + it('queries the 60-second window before start', async () => { + const client = createMockEsClient(hitsResponse(0)); + await checkPreExistingData(client, indices, start); + + expect(client.search).toHaveBeenCalledWith( + expect.objectContaining({ + index: indices, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: '2026-03-31T09:59:00.000Z', + lt: start, + }, + }, + }, + ], + }, + }, + }) + ); + }); + + it('returns false on no shards available error', async () => { + const error = new errors.ResponseError({ + statusCode: 503, + body: { + error: { + type: 'search_phase_execution_exception', + root_cause: [{ type: 'no_shard_available_action_exception' }], + }, + }, + headers: {}, + warnings: [], + meta: {} as never, + }); + const client = createMockEsClient(undefined, error); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); + + it('returns false on unexpected errors', async () => { + const client = createMockEsClient(undefined, new Error('Connection refused')); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts new file mode 100644 index 0000000000000..04a41df2f3e95 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { estypes } from '@elastic/elasticsearch'; + +const PRE_CHECK_WINDOW_MS = 60_000; + +/** + * Checks whether data was actively flowing into the given indices + * in the 60 seconds before `start`. If it was, time-range-based + * has-data detection is likely to produce false positives. + * + * Returns `false` on any error so it never blocks the main flow. + */ +export const checkPreExistingData = async ( + esClient: ElasticsearchClient, + indices: string[], + start: string +): Promise => { + try { + const startMs = new Date(start).getTime(); + const windowStart = new Date(startMs - PRE_CHECK_WINDOW_MS).toISOString(); + + const result = await esClient.search({ + index: indices, + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: windowStart, lt: start } } }], + }, + }, + }); + + return (result.hits.total as estypes.SearchTotalHits).value > 0; + } catch { + return false; + } +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts index 2aface3e84603..c72eec6dfd0b5 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts @@ -18,6 +18,7 @@ import { isNoShardsAvailableError, throwHasDataSearchError, } from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; export interface CreateCloudForwarderOnboardingFlowRouteResponse { onboardingId: string; @@ -79,28 +80,31 @@ const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ reason: 'Authorization is checked by Elasticsearch', }, }, - async handler(resources): Promise<{ hasData: boolean }> { + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { const { logType, start } = resources.params.query; const { elasticsearch } = await resources.context.core; const indexPattern = CLOUDFORWARDER_INDEX_PATTERNS[logType]; try { - const result = await elasticsearch.client.asCurrentUser.search({ - index: [indexPattern], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query: { - bool: { - filter: [{ range: { '@timestamp': { gte: start } } }], + const [preExisting, result] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, [indexPattern], start), + elasticsearch.client.asCurrentUser.search({ + index: [indexPattern], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: start } } }], + }, }, - }, - }); + }), + ]); const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; - return { hasData }; + return { hasData, hasPreExistingData: preExisting || undefined }; } catch (error) { if (isNoShardsAvailableError(error)) { return { hasData: false }; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index 2af0ecebe9190..7305c4eef9698 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -14,6 +14,7 @@ import { isNoShardsAvailableError, throwHasDataSearchError, } from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; import { resolveProbe } from './resolve_has_data_probes'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; @@ -38,6 +39,7 @@ export interface HasKubernetesDataRouteResponse { hasData: boolean; hasLogs?: boolean; hasMetrics?: boolean; + hasPreExistingData?: boolean; } const createKubernetesOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ @@ -168,13 +170,24 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, }; + const wiredStreamIndices = ['logs.otel*', 'logs.ecs*', 'metrics.otel*', 'metrics.ecs*']; + + // Check if data was already flowing into wired stream indices before + // the user started onboarding. If so, time-range detection on those + // indices would produce false positives, so we skip it. + const hasPreExistingData = start + ? await checkPreExistingData(elasticsearch.client.asCurrentUser, wiredStreamIndices, start) + : false; + // Wired streams (logs.otel*, logs.ecs*) use passthrough mapping where // onboarding.id is not indexed, so we cannot filter by it without a // runtime mapping (which times out on large clusters). Instead, fall - // back to a time-range-only query when a start time is provided. - const wiredStreamQuery: estypes.QueryDslQueryContainer | undefined = start - ? { bool: { filter: [{ range: { '@timestamp': { gte: start } } }] } } - : undefined; + // back to a time-range-only query when a start time is provided and + // no pre-existing data would cause false positives. + const wiredStreamQuery: estypes.QueryDslQueryContainer | undefined = + start && !hasPreExistingData + ? { bool: { filter: [{ range: { '@timestamp': { gte: start } } }] } } + : undefined; const searches: Array> = [ elasticsearch.client.asCurrentUser.search({ @@ -217,6 +230,7 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ hasData: hasLogs || hasMetrics, hasLogs, hasMetrics, + hasPreExistingData: hasPreExistingData || undefined, }; } catch (error) { if (isNoShardsAvailableError(error)) { diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index 49b497617a05c..a65509346ec61 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -16,6 +16,7 @@ import { isNoShardsAvailableError, throwHasDataSearchError, } from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; import { getAgentVersionInfo } from '../../lib/get_agent_version'; import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; @@ -93,10 +94,12 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ reason: 'Authorization is checked by Elasticsearch', }, }, - async handler(resources): Promise<{ hasData: boolean }> { + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { const { start, osType } = resources.params.query; const { elasticsearch } = await resources.context.core; + const allIndices = ['logs-*.otel-*', 'logs.otel', 'logs.otel.*', 'metrics-*.otel-*']; + const filters: estypes.QueryDslQueryContainer[] = [{ range: { '@timestamp': { gte: start } } }]; if (osType) { filters.push({ term: { 'host.os.type': osType } }); @@ -105,23 +108,26 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ bool: { filter: filters }, }; - const [logsResult, metricsResult] = await Promise.allSettled([ - elasticsearch.client.asCurrentUser.search({ - index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query, - }), - elasticsearch.client.asCurrentUser.search({ - index: ['metrics-*.otel-*'], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query, - }), + const [preExisting, [logsResult, metricsResult]] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, allIndices, start), + Promise.allSettled([ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*.otel-*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]), ]); const resolveProbe = (result: PromiseSettledResult): boolean => { @@ -137,7 +143,10 @@ const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ const hasLogs = resolveProbe(logsResult); const hasMetrics = resolveProbe(metricsResult); - return { hasData: hasLogs || hasMetrics }; + return { + hasData: hasLogs || hasMetrics, + hasPreExistingData: preExisting || undefined, + }; }, }); From 3ba7dd1700d22556e38f125712b30c31b483ad02 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 31 Mar 2026 12:47:32 +0000 Subject: [PATCH 17/21] Changes from node scripts/lint_ts_projects --fix --- .../plugins/observability_onboarding/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json b/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json index c9de5aeae6599..cac9b72d452fd 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json @@ -53,7 +53,8 @@ "@kbn/scout-synthtrace", "@kbn/synthtrace-client", "@kbn/streams-plugin", - "@kbn/data-views-plugin" + "@kbn/data-views-plugin", + "@kbn/core-elasticsearch-server" ], "exclude": ["target/**/*"] } From c4a01c301c54bc003de8d7a40b310d6b32559100 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 31 Mar 2026 12:59:41 +0000 Subject: [PATCH 18/21] Changes from node scripts/regenerate_moon_projects.js --update --- .../observability/plugins/observability_onboarding/moon.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml b/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml index 0a4084835df42..bf089c9ff3f3b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml +++ b/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml @@ -58,6 +58,7 @@ dependsOn: - '@kbn/synthtrace-client' - '@kbn/streams-plugin' - '@kbn/data-views-plugin' + - '@kbn/core-elasticsearch-server' tags: - plugin - prod From 1e991e546cb01d758d8cde70abc4db9751554f2f Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Wed, 1 Apr 2026 11:55:15 +0200 Subject: [PATCH 19/21] address comments, remove window blur action when preexisting data is there --- .../kubernetes/data_ingest_status.tsx | 29 ++++++++++-- .../quickstart_flows/kubernetes/index.tsx | 12 ++++- .../otel_kubernetes/otel_kubernetes_panel.tsx | 12 ++++- .../quickstart_flows/otel_logs/index.tsx | 46 +++++++++++-------- .../shared/use_pre_existing_data_check.ts | 44 ++++++++++++++++++ .../shared/use_time_window_data_detection.ts | 22 ++++++++- 6 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index 2b2e0850f3c7a..b71e27e4efea0 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -80,7 +80,9 @@ export function DataIngestStatus({ }, [isReady, hasPreExistingData, refetch, status]); useEffect(() => { - if (hasData === true && !dataReceivedTelemetrySent) { + if (dataReceivedTelemetrySent) return; + + if (hasData === true) { setDataReceivedTelemetrySent(true); analytics.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { flow_type: onboardingFlowType, @@ -88,21 +90,38 @@ export function DataIngestStatus({ step: 'logs-ingest', step_status: 'complete', }); + } else if (hasPreExistingData) { + setDataReceivedTelemetrySent(true); + analytics.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: onboardingFlowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'pre_existing_data', + }); } - }, [analytics, hasData, dataReceivedTelemetrySent, onboardingFlowType, onboardingId]); + }, [ + analytics, + hasData, + hasPreExistingData, + dataReceivedTelemetrySent, + onboardingFlowType, + onboardingId, + ]); // Notify parent when all required data types have arrived (not just any data). // This drives the step status to 'complete' and must wait for metrics // if any action link requires them. useEffect(() => { - if (isReady && !dataReceivedNotified) { + if ((isReady || hasPreExistingData) && !dataReceivedNotified) { onDataReceived?.(); setDataReceivedNotified(true); } - }, [isReady, onDataReceived, dataReceivedNotified]); + }, [isReady, hasPreExistingData, onDataReceived, dataReceivedNotified]); const isTroubleshootingVisible = - hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + hasData === false && + !hasPreExistingData && + Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; const filteredActionLinks = hasPreExistingData ? actionLinks diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx index b4ca9dad5204d..82ea42c069eea 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx @@ -19,6 +19,7 @@ import { CommandSnippet } from './command_snippet'; import { DataIngestStatus } from './data_ingest_status'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { useKubernetesFlow } from './use_kubernetes_flow'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { type IngestionMode } from '../shared/wired_streams_ingestion_selector'; @@ -52,12 +53,19 @@ export const KubernetesPanel: React.FC = () => { const [dataReceived, setDataReceived] = useState(false); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + const hasPreExistingDataEarly = usePreExistingDataCheck({ + flow: 'kubernetes', + onboardingId: data?.onboardingId, + }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'kubernetes', onboardingId: data?.onboardingId, }); + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + useEffect(() => { if (data) { onPageReady({ @@ -146,7 +154,7 @@ export const KubernetesPanel: React.FC = () => { defaultMessage: 'Monitor your Kubernetes cluster', } ), - status: (dataReceived + status: (dataReceived || hasPreExistingDataEarly ? 'complete' : isMonitoringStepActive ? 'current' diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx index 26a0c171ac773..388a941eadfb8 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx @@ -34,6 +34,7 @@ import { type ObservabilityOnboardingAppServices } from '../../..'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; import { FeedbackButtons } from '../shared/feedback_buttons'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { DataIngestStatus, type ActionLink } from '../kubernetes/data_ingest_status'; import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; @@ -89,11 +90,18 @@ export const OtelKubernetesPanel: React.FC = () => { const [dataReceived, setDataReceived] = useState(false); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + const hasPreExistingDataEarly = usePreExistingDataCheck({ + flow: 'kubernetes', + onboardingId: data?.onboardingId, + }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'kubernetes_otel', onboardingId: data?.onboardingId, }); + + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_OTEL_DATA_VIEW_SPEC } : {}; useEffect(() => { @@ -501,7 +509,7 @@ kubectl describe pod -n my-namespace`} defaultMessage: 'Visualize your data', } ), - status: (dataReceived + status: (dataReceived || hasPreExistingDataEarly ? 'complete' : isMonitoringStepActive ? 'current' diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index f6fea1eeadd89..b6e4a28bdf84e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -32,6 +32,7 @@ import { usePerformanceContext } from '@kbn/ebt-tools'; import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; import type { ObservabilityOnboardingAppServices } from '../../..'; import { useFetcher } from '../../../hooks/use_fetcher'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; import { ProgressIndicator } from '../shared/progress_indicator'; @@ -97,15 +98,17 @@ export const OtelLogsPanel: React.FC = () => { const [selectedTab, setSelectedTab] = useState('linux'); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + const hasPreExistingDataEarly = usePreExistingDataCheck({ flow: 'otel_host' }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ isActive: !!setupData, onboardingFlowType: 'otel_logs', onboardingId: setupData?.onboardingId, }); - // Set sessionStartTime when monitoring begins (first blur) rather than on - // mount, to narrow the time-window and reduce false positives from other - // OTel collectors already ingesting data on the same cluster. + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + + // Set sessionStartTime when monitoring begins, not on mount. const [sessionStartTime, setSessionStartTime] = useState(null); useEffect(() => { if (isMonitoringStepActive && sessionStartTime === null) { @@ -123,6 +126,8 @@ export const OtelLogsPanel: React.FC = () => { endpoint: '/internal/observability_onboarding/otel_host/has-data', }); + const hasPreExistingDataFinal = hasPreExistingData || hasPreExistingDataEarly; + const isMetricsOnboardingEnabled = usePricingFeature( ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING ); @@ -427,14 +432,14 @@ export const OtelLogsPanel: React.FC = () => { defaultMessage: 'Visualize your data', } ), - status: (hasData || hasPreExistingData + status: (hasData || hasPreExistingDataFinal ? 'complete' : isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> - {!(hasPreExistingData && !hasData) && ( + {!(hasPreExistingDataFinal && !hasData) && ( { )} - {(hasData === true || hasPreExistingData) && visualizeActionLinks.length > 0 && ( - <> - - - - )} + {(hasData === true || hasPreExistingDataFinal) && + visualizeActionLinks.length > 0 && ( + <> + + + + )} ) : null, }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts new file mode 100644 index 0000000000000..398e2cd2e4b8b --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState } from 'react'; +import { useFetcher } from '../../../hooks/use_fetcher'; + +const FLOW_ENDPOINTS = { + kubernetes: '/internal/observability_onboarding/kubernetes/{onboardingId}/has-data', + otel_host: '/internal/observability_onboarding/otel_host/has-data', +} as const; + +type PreExistingDataFlow = keyof typeof FLOW_ENDPOINTS; + +export function usePreExistingDataCheck({ + flow, + onboardingId, +}: { + flow: PreExistingDataFlow; + onboardingId?: string; +}): boolean { + const endpoint = FLOW_ENDPOINTS[flow]; + const needsOnboardingId = flow === 'kubernetes'; + const [start] = useState(() => new Date().toISOString()); + + const { data } = useFetcher( + (callApi): Promise<{ hasPreExistingData?: boolean }> | undefined => { + if (needsOnboardingId && !onboardingId) return; + return callApi(`GET ${endpoint}` as Parameters[0], { + params: { + ...(onboardingId ? { path: { onboardingId } } : {}), + query: { start }, + }, + }); + }, + [endpoint, start, onboardingId, needsOnboardingId], + { showToastOnError: false } + ); + + return data?.hasPreExistingData ?? false; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts index 71687c017838f..154b6fb4a78ae 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts @@ -106,7 +106,9 @@ export function useTimeWindowDataDetection({ ]); useEffect(() => { - if (hasDataResponse?.hasData === true && !dataReceivedTelemetrySent) { + if (dataReceivedTelemetrySent) return; + + if (hasDataResponse?.hasData === true) { setDataReceivedTelemetrySent(true); analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { flow_type: flowType, @@ -114,8 +116,23 @@ export function useTimeWindowDataDetection({ step: 'logs-ingest', step_status: 'complete', }); + } else if (hasDataResponse?.hasPreExistingData === true) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: flowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'pre_existing_data', + }); } - }, [analytics, hasDataResponse?.hasData, dataReceivedTelemetrySent, flowType, onboardingId]); + }, [ + analytics, + hasDataResponse?.hasData, + hasDataResponse?.hasPreExistingData, + dataReceivedTelemetrySent, + flowType, + onboardingId, + ]); // Treat both "hasData === false" and fetch failures (where hasDataResponse // is undefined but the request completed) as "no data yet" for the purpose @@ -127,6 +144,7 @@ export function useTimeWindowDataDetection({ const isTroubleshootingVisible = isMonitoringActive && noDataConfirmed && + !hasDataResponse?.hasPreExistingData && checkDataStartTime !== null && Date.now() - checkDataStartTime > troubleshootingDelay; From f076ee05f6cfe5f24d9c7dcbafc7bdd04f37eb19 Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Thu, 2 Apr 2026 12:57:59 +0200 Subject: [PATCH 20/21] widen pre-existing data window and gate bypass on wired streams --- .../quickstart_flows/kubernetes/data_ingest_status.tsx | 4 +++- .../otel_kubernetes/otel_kubernetes_panel.tsx | 2 ++ .../quickstart_flows/shared/use_pre_existing_data_check.ts | 5 ++++- .../server/lib/check_pre_existing_data.test.ts | 4 ++-- .../server/lib/check_pre_existing_data.ts | 4 ++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index b71e27e4efea0..5745a6f1cebe1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -26,6 +26,7 @@ interface Props { integration: string; actionLinks: ActionLink[]; onDataReceived?: () => void; + respectPreExistingData?: boolean; } const FETCH_INTERVAL = 2000; @@ -38,6 +39,7 @@ export function DataIngestStatus({ integration, actionLinks, onDataReceived, + respectPreExistingData = true, }: Props) { const [checkDataStartTime] = useState(Date.now()); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); @@ -60,7 +62,7 @@ export function DataIngestStatus({ const hasData = data?.hasData ?? false; const hasLogs = data?.hasLogs ?? hasData; const hasMetrics = data?.hasMetrics ?? hasData; - const hasPreExistingData = data?.hasPreExistingData ?? false; + const hasPreExistingData = respectPreExistingData ? data?.hasPreExistingData ?? false : false; const needsMetrics = actionLinks.some((actionLink) => actionLink.requires === 'metrics'); const isReady = needsMetrics ? hasMetrics : hasData; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx index 388a941eadfb8..25f8e94df1b7b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx @@ -93,6 +93,7 @@ export const OtelKubernetesPanel: React.FC = () => { const hasPreExistingDataEarly = usePreExistingDataCheck({ flow: 'kubernetes', onboardingId: data?.onboardingId, + enabled: useWiredStreams, }); const windowBlurred = useWindowBlurDataMonitoringTrigger({ @@ -522,6 +523,7 @@ kubectl describe pod -n my-namespace`} integration="kubernetes_otel" actionLinks={otelKubernetesActionLinks} onDataReceived={() => setDataReceived(true)} + respectPreExistingData={useWiredStreams} /> ), }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts index 398e2cd2e4b8b..f5e2f1cff425f 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts @@ -18,9 +18,11 @@ type PreExistingDataFlow = keyof typeof FLOW_ENDPOINTS; export function usePreExistingDataCheck({ flow, onboardingId, + enabled = true, }: { flow: PreExistingDataFlow; onboardingId?: string; + enabled?: boolean; }): boolean { const endpoint = FLOW_ENDPOINTS[flow]; const needsOnboardingId = flow === 'kubernetes'; @@ -28,6 +30,7 @@ export function usePreExistingDataCheck({ const { data } = useFetcher( (callApi): Promise<{ hasPreExistingData?: boolean }> | undefined => { + if (!enabled) return; if (needsOnboardingId && !onboardingId) return; return callApi(`GET ${endpoint}` as Parameters[0], { params: { @@ -36,7 +39,7 @@ export function usePreExistingDataCheck({ }, }); }, - [endpoint, start, onboardingId, needsOnboardingId], + [endpoint, start, onboardingId, needsOnboardingId, enabled], { showToastOnError: false } ); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts index 4b1ac081fe42e..6fdad24a31827 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts @@ -31,7 +31,7 @@ describe('checkPreExistingData', () => { expect(await checkPreExistingData(client, indices, start)).toBe(false); }); - it('queries the 60-second window before start', async () => { + it('queries the 5-minute window before start', async () => { const client = createMockEsClient(hitsResponse(0)); await checkPreExistingData(client, indices, start); @@ -46,7 +46,7 @@ describe('checkPreExistingData', () => { { range: { '@timestamp': { - gte: '2026-03-31T09:59:00.000Z', + gte: '2026-03-31T09:55:00.000Z', lt: start, }, }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts index 04a41df2f3e95..f86166c63a046 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts @@ -8,11 +8,11 @@ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { estypes } from '@elastic/elasticsearch'; -const PRE_CHECK_WINDOW_MS = 60_000; +const PRE_CHECK_WINDOW_MS = 300_000; /** * Checks whether data was actively flowing into the given indices - * in the 60 seconds before `start`. If it was, time-range-based + * in the 5 minutes before `start`. If it was, time-range-based * has-data detection is likely to produce false positives. * * Returns `false` on any error so it never blocks the main flow. From 477be11d42e889b26c0e6a5a42b113e58a97215d Mon Sep 17 00:00:00 2001 From: Robert Stelmach Date: Tue, 7 Apr 2026 14:07:22 +0200 Subject: [PATCH 21/21] add precheck for APM otel flow --- .../quickstart_flows/otel_apm/index.tsx | 58 +++++++++++-------- .../shared/use_pre_existing_data_check.ts | 1 + .../server/routes/otel_apm/route.ts | 42 ++++++++------ 3 files changed, 59 insertions(+), 42 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index 1213faef688ff..7c9b8dedea3a1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -33,6 +33,7 @@ import type { ObservabilityOnboardingAppServices } from '../../..'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; import { ProgressIndicator } from '../shared/progress_indicator'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { FeedbackButtons } from '../shared/feedback_buttons'; @@ -67,15 +68,20 @@ export function OtelApmQuickstartFlow() { } }, [data, onPageReady]); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + const hasPreExistingDataEarly = usePreExistingDataCheck({ flow: 'otel_apm' }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'otel_apm', onboardingId: data?.onboardingId, }); - // Set sessionStartTime when monitoring begins (first blur) rather than on - // mount, to narrow the time-window and reduce false positives from other - // APM services already ingesting data on the same cluster. + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + + // Set sessionStartTime when monitoring begins (first blur or early + // pre-existing data detection) rather than on mount, to narrow the + // time-window and reduce false positives from other APM services + // already ingesting data on the same cluster. const [sessionStartTime, setSessionStartTime] = useState(null); useEffect(() => { if (isMonitoringStepActive && sessionStartTime === null) { @@ -84,7 +90,7 @@ export function OtelApmQuickstartFlow() { }, [isMonitoringStepActive, sessionStartTime]); const trimmedServiceName = serviceName.trim(); - const { hasData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, sessionStartTime: sessionStartTime ?? '', fetchInterval: FETCH_INTERVAL, @@ -95,6 +101,8 @@ export function OtelApmQuickstartFlow() { extraQueryParams: trimmedServiceName ? { serviceName: trimmedServiceName } : undefined, }); + const hasPreExistingDataFinal = hasPreExistingData || hasPreExistingDataEarly; + if (error !== undefined) { return ; } @@ -147,30 +155,32 @@ export function OtelApmQuickstartFlow() { title: i18n.translate('xpack.observability_onboarding.otelApm.monitorStepTitle', { defaultMessage: 'Visualize your data', }), - status: (hasData + status: (hasData || hasPreExistingDataFinal ? 'complete' : isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive ? ( <> - + {!hasPreExistingDataFinal && ( + + )} {isTroubleshootingVisible && ( <> @@ -199,7 +209,7 @@ export function OtelApmQuickstartFlow() { )} - {hasData === true && ( + {(hasData === true || hasPreExistingDataFinal) && ( <> { + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { const { start, serviceName } = resources.params.query; const { elasticsearch } = await resources.context.core; + const apmIndices = [ + 'traces-apm*', + 'traces-*.otel-*', + 'logs-apm*', + 'logs-*.otel-*', + 'metrics-apm*', + 'metrics-*.otel-*', + 'apm-*', + ]; + try { const filters: estypes.QueryDslQueryContainer[] = [ { range: { '@timestamp': { gte: start } } }, @@ -83,25 +94,20 @@ const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ bool: { filter: filters }, }; - const result = await elasticsearch.client.asCurrentUser.search({ - index: [ - 'traces-apm*', - 'traces-*.otel-*', - 'logs-apm*', - 'logs-*.otel-*', - 'metrics-apm*', - 'metrics-*.otel-*', - 'apm-*', - ], - ignore_unavailable: true, - allow_partial_search_results: true, - size: 0, - terminate_after: 1, - query, - }); + const [preExisting, result] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, apmIndices, start), + elasticsearch.client.asCurrentUser.search({ + index: apmIndices, + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]); const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; - return { hasData }; + return { hasData, hasPreExistingData: preExisting || undefined }; } catch (error) { if (isNoShardsAvailableError(error)) { return { hasData: false };