diff --git a/x-pack/platform/plugins/private/translations/translations/de-DE.json b/x-pack/platform/plugins/private/translations/translations/de-DE.json index 88b6dc3e37fcb..1b287b08a5ec9 100644 --- a/x-pack/platform/plugins/private/translations/translations/de-DE.json +++ b/x-pack/platform/plugins/private/translations/translations/de-DE.json @@ -29032,7 +29032,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumentieren Sie Ihre Anwendung (optional)", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Überprüfen Sie die Gesundheit Ihres Kubernetes-Clusters:", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Daten visualisieren", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysieren Sie die Gesundheit Ihres Kubernetes-Clusters und überwachen Sie Ihre Container-Workloads.", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "Weitere Informationen finden Sie in der Dokumentation", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Wählen Sie eine Programmiersprache aus.", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Erkunden Sie das Service-Inventar", @@ -29043,9 +29042,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "Fehler bei der Installation der Integration", "xpack.observability_onboarding.otelLogs.status.failedDetails": "Eingehende Daten könnten möglicherweise nicht korrekt indiziert werden. Details:", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Plattform wählen", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Dokumentation öffnen", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Logs durchsuchen", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Offene Hosts", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Ab dem Setup werden neue log Meldungen gesammelt.", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Der Standardlogpfad ist /var/log/*. Sie können diesen Pfad bei Bedarf in der Datei otel.yml ändern.", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informationen zur Konfiguration", @@ -29056,10 +29052,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Daten visualisieren", "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.label": "Technische Vorschau", "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.tooltip": "Diese Funktionalität befindet sich in der technischen Vorschau und kann in einer zukünftigen Version geändert oder vollständig entfernt werden. Elastic wird sich bemühen, alle Probleme zu beheben, aber die Features in der technischen Vorschau unterliegen nicht dem Support-SLA der offiziellen GA-Features.", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Weitere Einzelheiten und Lösungen zur Fehlerbehebung finden Sie in unserer Dokumentation. {link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Ihre Metriken anzeigen und analysieren", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Ihre Logs ansehen und analysieren", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Kehren Sie nach der Ausführung des vorherigen Befehls zurück und sehen Sie sich Ihre Daten an.", "xpack.observability_onboarding.otelTile.description": "Erfassen Sie Protokolle und Host-Metriken mit der Elastic-Distribution des OpenTelemetry Collector", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "Laden Sie Daten aus einer CSV-, TSV-, JSON- oder anderen Log-Datei zur Analyse in Elasticsearch hoch.", diff --git a/x-pack/platform/plugins/private/translations/translations/fr-FR.json b/x-pack/platform/plugins/private/translations/translations/fr-FR.json index 1a7ae5178b5a7..025658cba12b9 100644 --- a/x-pack/platform/plugins/private/translations/translations/fr-FR.json +++ b/x-pack/platform/plugins/private/translations/translations/fr-FR.json @@ -29381,7 +29381,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumenter votre application (facultatif)", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Vérifiez l'intégrité de votre cluster Kubernetes :", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Visualiser vos données", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysez l'intégrité de votre cluster Kubernetes et monitorez les charges de travail de vos conteneurs.", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "se reporter à la documentation", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Sélectionner un langage de programmation", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Explorer l'inventaire des services", @@ -29392,9 +29391,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "Échec de l'installation de l'intégration", "xpack.observability_onboarding.otelLogs.status.failedDetails": "Les données entrantes peuvent ne pas être indexées correctement. Détails :", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Choisissez une plateforme", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Ouvrir la documentation", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Explorer les logs", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Ouvrir les hôtes", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Les nouveaux messages de log sont collectés à partir de la configuration.", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Le chemin des logs par défaut est /var/log/*. Vous pouvez si nécessaire modifier ce chemin dans le fichier otel.yml.", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informations sur la configuration", @@ -29403,10 +29399,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "Sélectionnez votre plateforme", "xpack.observability_onboarding.otelLogsPanel.steps.start": "Lancez le collecteur", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Visualiser vos données", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Vous trouverez plus d'informations et une solution de résolution des problèmes dans notre documentation. {link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Visualisez et analysez vos métriques", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Visualisez et analysez vos logs", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Après avoir exécuté la commande précédente, revenez et visualisez vos données.", "xpack.observability_onboarding.otelTile.description": "Collectez les logs et les indicateurs de l'hôte à l'aide de la distribution Elastic du collecteur OpenTelemetry", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "Téléchargez les données d'un fichier CSV, TSV, JSON ou autre fichier log vers Elasticsearch pour analyse.", diff --git a/x-pack/platform/plugins/private/translations/translations/ja-JP.json b/x-pack/platform/plugins/private/translations/translations/ja-JP.json index 75c9c606d8a8b..1b1b4275a336a 100644 --- a/x-pack/platform/plugins/private/translations/translations/ja-JP.json +++ b/x-pack/platform/plugins/private/translations/translations/ja-JP.json @@ -29435,7 +29435,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "アプリケーションのインストルメンテーション（任意）", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Kubernetesクラスター正常性を確認：", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "データを可視化する", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Kubernetesクラスターの正常性を分析し、コンテナーのワークロードを監視します。", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "ドキュメントを参照", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "プログラミング言語を選択", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "サービスインベントリを探索", @@ -29446,9 +29445,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "統合のインストールに失敗しました", "xpack.observability_onboarding.otelLogs.status.failedDetails": "受信データは正しくインデックス化されていない可能性があります。詳細：", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "プラットフォームを選択", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "ドキュメントを開く", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "ログを探索", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "ホストを開く", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "今後、新しいログメッセージはセットアップから収集されます。", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "デフォルトのログのパスは/var/log/*です。必要に応じて、otel.ymlファイルでこのパスを変更できます。", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "構成情報", @@ -29457,10 +29453,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "プラットフォームを選択", "xpack.observability_onboarding.otelLogsPanel.steps.start": "コレクターを開始", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "データを可視化する", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "詳細とトラブルシューティングの解決策については、ドキュメントをご覧ください。{link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "メトリックを表示して分析", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "ログを表示して分析", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "前のコマンドを実行した後、データに戻って表示します。", "xpack.observability_onboarding.otelTile.description": "OpenTelemetryコレクターのElasticディストリビューションを使用して、ログとホストメトリックを収集します。", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "分析するため、CSV、TSV、JSON、他のログファイルからElasticsearchにアップロードします。", diff --git a/x-pack/platform/plugins/private/translations/translations/zh-CN.json b/x-pack/platform/plugins/private/translations/translations/zh-CN.json index d1f42e4429647..a2d5319d9e046 100644 --- a/x-pack/platform/plugins/private/translations/translations/zh-CN.json +++ b/x-pack/platform/plugins/private/translations/translations/zh-CN.json @@ -29414,7 +29414,6 @@ "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "检测您的应用程序（可选）", "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "检查 Kubernetes 集群的运行状况：", "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "可视化数据", - "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "分析 Kubernetes 集群的运行状况并监测容器工作负载。", "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "参阅文档", "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "选择编程语言", "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "浏览服务库存", @@ -29425,9 +29424,6 @@ "xpack.observability_onboarding.otelLogs.status.failed": "集成安装失败", "xpack.observability_onboarding.otelLogs.status.failedDetails": "传入数据可能未正确索引。详情：", "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "选择平台", - "xpack.observability_onboarding.otelLogsPanel.documentationLink": "打开文档", - "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "浏览日志", - "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "打开主机", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "将从设置完成后收集新的日志消息。", "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "默认日志路径为 /var/log/*。如果需要，可以在 otel.yml 文件中更改此路径。", "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "配置信息", @@ -29436,10 +29432,6 @@ "xpack.observability_onboarding.otelLogsPanel.steps.platform": "选择平台", "xpack.observability_onboarding.otelLogsPanel.steps.start": "启动收集器", "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "可视化数据", - "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "在我们的文档中查找更多详情和故障排除解决方案。{link}", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "查看并分析您的指标", - "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "查看并分析您的日志", - "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "运行上一个命令后，返回并查看您的数据。", "xpack.observability_onboarding.otelTile.description": "使用 OpenTelemetry 收集器的 Elastic 发行版收集日志和主机指标", "xpack.observability_onboarding.otelTile.title": "OpenTelemetry", "xpack.observability_onboarding.packageList.uploadFileDescription": "从 CSV、TSV、JSON 或其他日志文件上传数据到 Elasticsearch 以进行分析。", diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts index a236b06f9e9a3..7193d2c6efb9d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts @@ -27,3 +27,9 @@ export const CLOUDFORMATION_STACK_CONFIGS = { } as const; export type LogType = keyof typeof CLOUDFORMATION_STACK_CONFIGS; + +export const CLOUDFORWARDER_INDEX_PATTERNS: Record = { + vpcflow: 'logs-aws.vpcflow.otel-*', + elbaccess: 'logs-aws.elbaccess.otel-*', + cloudtrail: 'logs-aws.cloudtrail.otel-*', +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts index 61c2aa711b25f..41543157122c0 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts @@ -57,12 +57,23 @@ test('Otel Host', async ({ fs.writeFileSync(outputPath, codeSnippet); /** - * There is no explicit data ingest indication - * in the flow, so we need to rely on a timeout. - * 3 minutes should be enough for the collector - * to initialize and start ingesting data. + * The page waits for the browser window to lose + * focus as a signal to start checking for incoming data */ - await page.waitForTimeout(3 * 60000); + await page.evaluate('window.dispatchEvent(new Event("blur"))'); + + /** + * Wait for the data received indicator to appear. + * The flow polls for data after the blur event and + * shows "We are monitoring your host" once data arrives. + */ + await otelHostFlowPage.assertDataReceivedIndicator(); + + /** + * Additional buffer to ensure data has propagated + * to dashboards and Discover before navigating. + */ + await page.waitForTimeout(2 * 60000); /** * Wired streams only reroutes logs (to logs.otel); metrics and traces are diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts index 2247b3c29bb4d..e5a391d3c682e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts @@ -89,12 +89,24 @@ test('Otel Kubernetes', async ({ fs.writeFileSync(outputPath, codeSnippet); /** - * There is no explicit data ingest indication - * in the flow, so we need to rely on a timeout. - * 5 minutes should be enough for the stack to be - * created and to start pushing data. + * The page waits for the browser window to lose + * focus as a signal to start checking for incoming data */ - await page.waitForTimeout(5 * 60000); + await page.evaluate('window.dispatchEvent(new Event("blur"))'); + + /** + * Wait for the data received indicator to appear. + * The flow now uses DataIngestStatus which polls for data + * after the blur event and shows "We are monitoring your cluster" + * once both logs and metrics have arrived. + */ + await otelKubernetesFlowPage.assertDataReceivedIndicator(); + + /** + * Additional buffer to ensure data has propagated + * to dashboards and Discover before navigating. + */ + await page.waitForTimeout(2 * 60000); /** * Wired streams only reroutes logs (to logs.otel); metrics and traces are diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts index 339ad7da04160..84704894353c1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts @@ -12,12 +12,20 @@ export class OtelHostFlowPage { private readonly exploreLogsButton: Locator; private readonly exploreMetricsButton: Locator; + private readonly dataReceivedIndicator: Locator; constructor(page: Page) { this.page = page; - this.exploreLogsButton = this.page.getByTestId('obltOnboardingExploreLogs'); - this.exploreMetricsButton = this.page.getByTestId('obltOnboardingExploreMetrics'); + this.exploreLogsButton = this.page.getByTestId( + 'observabilityOnboardingDataIngestStatusActionLink-logs' + ); + this.exploreMetricsButton = this.page.getByTestId( + 'observabilityOnboardingDataIngestStatusActionLink-metrics' + ); + this.dataReceivedIndicator = this.page + .getByTestId('observabilityOnboardingOtelHostDataProgressIndicator') + .getByText('We are monitoring your host'); } public async selectPlatform(osName: string) { @@ -56,6 +64,13 @@ export class OtelHostFlowPage { await this.exploreLogsButton.click(); } + public async assertDataReceivedIndicator(): Promise { + await expect( + this.dataReceivedIndicator, + 'Data received indicator should be visible' + ).toBeVisible({ timeout: 5 * 60_000 }); + } + public async assertLogsExplorationButtonVisible() { await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible(); } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts index 3828dacef0caa..1b604614da30e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts @@ -12,12 +12,16 @@ export class OtelKubernetesFlowPage { context: BrowserContext; private readonly exploreLogsButton: Locator; + private readonly dataReceivedIndicator: Locator; constructor(page: Page, context: BrowserContext) { this.page = page; this.context = context; this.exploreLogsButton = this.page.getByText('Explore logs'); + this.dataReceivedIndicator = this.page + .getByTestId('observabilityOnboardingKubernetesPanelDataProgressIndicator') + .getByText('We are monitoring your cluster'); } public async copyHelmRepositorySnippetToClipboard() { @@ -82,6 +86,13 @@ export class OtelKubernetesFlowPage { } } + public async assertDataReceivedIndicator(): Promise { + await expect( + this.dataReceivedIndicator, + 'Data received indicator should be visible' + ).toBeVisible({ timeout: 5 * 60_000 }); + } + public async assertLogsExplorationButtonVisible() { await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible(); } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml b/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml index c568dcfcd902e..d176bb63cb93d 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml +++ b/x-pack/solutions/observability/plugins/observability_onboarding/moon.yml @@ -60,6 +60,7 @@ dependsOn: - '@kbn/data-views-plugin' - '@kbn/ingest-hub-plugin' - '@kbn/shared-ux-utility' + - '@kbn/core-elasticsearch-server' tags: - plugin - prod diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx index f3668bac3722a..6da361480f869 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/cloudforwarder/index.tsx @@ -6,10 +6,10 @@ */ import React, { useEffect, useState } from 'react'; +import { css } from '@emotion/react'; import { EuiButton, EuiButtonGroup, - EuiCallOut, EuiFieldText, EuiFormRow, EuiLink, @@ -20,6 +20,7 @@ import { EuiSteps, EuiText, } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { usePerformanceContext } from '@kbn/ebt-tools'; @@ -28,6 +29,10 @@ import type { ObservabilityOnboardingAppServices } from '../../..'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; +import { ProgressIndicator } from '../shared/progress_indicator'; +import { GetStartedPanel } from '../shared/get_started_panel'; import { useCloudForwarderFlow } from './use_cloudforwarder_flow'; import { EmptyPrompt } from '../shared/empty_prompt'; import { OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; @@ -37,6 +42,9 @@ import { isValidS3BucketName, buildS3BucketArn, buildCloudFormationUrl } from '. const EDOT_CLOUD_FORWARDER_DOCS_URL = 'https://www.elastic.co/docs/reference/opentelemetry/edot-cloud-forwarder/aws'; +const FETCH_INTERVAL = 5000; +const SHOW_TROUBLESHOOTING_DELAY = 300_000; + export function CloudForwarderPanel() { useFlowBreadcrumb({ text: i18n.translate( @@ -49,6 +57,7 @@ export function CloudForwarderPanel() { const { services: { + http, analytics, context: { cloudServiceProvider }, }, @@ -62,6 +71,28 @@ export function CloudForwarderPanel() { trimmedBucketName.length > 0 && !isValidS3BucketName(trimmedBucketName); const { onPageReady } = usePerformanceContext(); + const [sessionStartTime, setSessionStartTime] = useState(null); + const [monitoringLogType, setMonitoringLogType] = useState(null); + const [launchStackClicked, setLaunchStackClicked] = useState(false); + + const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS && launchStackClicked, + onboardingFlowType: 'cloudforwarder', + onboardingId: data?.onboardingId, + }); + + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: + isMonitoringStepActive && monitoringLogType !== null && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'cloudforwarder', + onboardingId: data?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/cloudforwarder/has-data', + extraQueryParams: monitoringLogType ? { logType: monitoringLogType } : undefined, + }); + useEffect(() => { if (data) { onPageReady({ @@ -216,6 +247,7 @@ export function CloudForwarderPanel() { value={s3BucketName} onChange={(e) => setS3BucketName(e.target.value)} isInvalid={isBucketNameInvalid} + disabled={launchStackClicked} placeholder={i18n.translate( 'xpack.observability_onboarding.cloudforwarderPanel.s3BucketNamePlaceholder', { @@ -246,6 +278,7 @@ export function CloudForwarderPanel() { idSelected={selectedLogType} onChange={(id) => setSelectedLogType(id as LogType)} buttonSize="m" + isDisabled={launchStackClicked} /> )} @@ -290,6 +323,9 @@ export function CloudForwarderPanel() { fill isDisabled={!isValidS3BucketName(trimmedBucketName)} onClick={() => { + setLaunchStackClicked(true); + setMonitoringLogType(selectedLogType); + setSessionStartTime(new Date().toISOString()); analytics?.reportEvent( OBSERVABILITY_ONBOARDING_FLOW_PROGRESS_TELEMETRY_EVENT.eventType, { @@ -323,35 +359,90 @@ export function CloudForwarderPanel() { defaultMessage: 'Visualize your data', } ), - children: ( - -

- - {i18n.translate( - 'xpack.observability_onboarding.cloudforwarderPanel.strong.logsawsLabel', - { defaultMessage: 'logs-aws.*' } - )} - - ), - }} + status: (hasData || hasPreExistingData + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive ? ( + <> + {!(hasPreExistingData && !hasData) && ( + -

- - ), + )} + + {isTroubleshootingVisible && ( + <> + + + + {i18n.translate( + 'xpack.observability_onboarding.cloudforwarderPanel.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } + )} + + ), + }} + /> + + + )} + + {(hasData === true || hasPreExistingData) && ( + <> + + + + )} + + ) : null, }, ]; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx index e3878af51efc7..5745a6f1cebe1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/data_ingest_status.tsx @@ -11,54 +11,66 @@ import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; -import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; -import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; import { ProgressIndicator } from '../shared/progress_indicator'; -import { GetStartedPanel } from '../shared/get_started_panel'; +import { GetStartedPanel, type ActionLink } from '../shared/get_started_panel'; import type { ObservabilityOnboardingContextValue } from '../../../plugin'; -import { usePricingFeature } from '../shared/use_pricing_feature'; -import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; -import { type IngestionMode } from '../shared/wired_streams_ingestion_selector'; -import { WIRED_ECS_DATA_VIEW_SPEC } from '../shared/wired_streams_data_view'; + +export type { ActionLink }; interface Props { onboardingId: string; - ingestionMode: IngestionMode; + onboardingFlowType: string; + dataset: string; + integration: string; + actionLinks: ActionLink[]; + onDataReceived?: () => void; + respectPreExistingData?: boolean; } const FETCH_INTERVAL = 2000; const SHOW_TROUBLESHOOTING_DELAY = 120000; // 2 minutes -const CLUSTER_OVERVIEW_DASHBOARD_ID = 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'; -export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { - const metricsOnboardingEnabled = usePricingFeature( - ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING - ); +export function DataIngestStatus({ + onboardingId, + onboardingFlowType, + dataset, + integration, + actionLinks, + onDataReceived, + respectPreExistingData = true, +}: Props) { const [checkDataStartTime] = useState(Date.now()); const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const [dataReceivedNotified, setDataReceivedNotified] = useState(false); const { - services: { share, analytics }, + services: { analytics }, } = useKibana(); - const dashboardLocator = share.url.locators.get(DASHBOARD_APP_LOCATOR); - const logsLocator = share.url.locators.get(LOGS_LOCATOR_ID); - const useWiredStreams = ingestionMode === 'wired'; - const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_ECS_DATA_VIEW_SPEC } : {}; + + const startIso = new Date(checkDataStartTime).toISOString(); const { data, status, refetch } = useFetcher( (callApi) => { return callApi('GET /internal/observability_onboarding/kubernetes/{onboardingId}/has-data', { - params: { path: { onboardingId } }, + params: { path: { onboardingId }, query: { start: startIso } }, }); }, - [onboardingId] + [onboardingId, startIso] ); + const hasData = data?.hasData ?? false; + const hasLogs = data?.hasLogs ?? hasData; + const hasMetrics = data?.hasMetrics ?? hasData; + const hasPreExistingData = respectPreExistingData ? data?.hasPreExistingData ?? false : false; + + const needsMetrics = actionLinks.some((actionLink) => actionLink.requires === 'metrics'); + const isReady = needsMetrics ? hasMetrics : hasData; + useEffect(() => { const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; - if (pendingStatusList.includes(status) || data?.hasData === true) { + if (pendingStatusList.includes(status) || isReady || hasPreExistingData) { return; } @@ -67,34 +79,107 @@ export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { }, FETCH_INTERVAL); return () => clearTimeout(timeout); - }, [data?.hasData, refetch, status]); + }, [isReady, hasPreExistingData, refetch, status]); useEffect(() => { - if (data?.hasData === true && !dataReceivedTelemetrySent) { + if (dataReceivedTelemetrySent) return; + + if (hasData === true) { setDataReceivedTelemetrySent(true); analytics.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { - flow_type: 'kubernetes', + flow_type: onboardingFlowType, flow_id: onboardingId, step: 'logs-ingest', step_status: 'complete', }); + } else if (hasPreExistingData) { + setDataReceivedTelemetrySent(true); + analytics.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: onboardingFlowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'pre_existing_data', + }); + } + }, [ + analytics, + hasData, + hasPreExistingData, + dataReceivedTelemetrySent, + onboardingFlowType, + onboardingId, + ]); + + // Notify parent when all required data types have arrived (not just any data). + // This drives the step status to 'complete' and must wait for metrics + // if any action link requires them. + useEffect(() => { + if ((isReady || hasPreExistingData) && !dataReceivedNotified) { + onDataReceived?.(); + setDataReceivedNotified(true); } - }, [analytics, data?.hasData, dataReceivedTelemetrySent, onboardingId]); + }, [isReady, hasPreExistingData, onDataReceived, dataReceivedNotified]); const isTroubleshootingVisible = - data?.hasData === false && Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + hasData === false && + !hasPreExistingData && + Date.now() - checkDataStartTime > SHOW_TROUBLESHOOTING_DELAY; + + const filteredActionLinks = hasPreExistingData + ? actionLinks + : actionLinks.filter((actionLink) => { + const requires = actionLink.requires ?? 'any'; + + if (requires === 'logs') { + return hasLogs; + } + + if (requires === 'metrics') { + return hasMetrics; + } + + return hasData; + }); + + const filteredActionLinksWithHref = filteredActionLinks.filter((actionLink) => + Boolean(actionLink.href) + ); + + const progressTitle = (() => { + if (hasData && needsMetrics && !hasMetrics) { + return i18n.translate( + 'xpack.observability_onboarding.dataIngestStatus.waitingForMetricsTitle', + { defaultMessage: 'Waiting for metrics to be shipped' } + ); + } + + if (hasData) { + return i18n.translate( + 'xpack.observability_onboarding.dataIngestStatus.monitoringClusterTitle', + { + defaultMessage: 'We are monitoring your cluster', + } + ); + } + + return i18n.translate('xpack.observability_onboarding.dataIngestStatus.waitingForDataTitle', { + defaultMessage: 'Waiting for data to be shipped', + }); + })(); return ( <> - + {!(hasPreExistingData && !hasData) && ( + + )} {isTroubleshootingVisible && ( <> @@ -125,53 +210,18 @@ export function DataIngestStatus({ onboardingId, ingestionMode }: Props) { )} - {data?.hasData === true && ( + {(hasData === true || hasPreExistingData) && filteredActionLinksWithHref.length > 0 && ( <> )} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx index 0d44bb394bca9..9abb33d33f237 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx @@ -9,27 +9,57 @@ import React, { useEffect, useState } from 'react'; import type { EuiStepStatus } from '@elastic/eui'; import { EuiPanel, EuiSkeletonRectangle, EuiSkeletonText, EuiSpacer, EuiSteps } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; import { usePerformanceContext } from '@kbn/ebt-tools'; +import { DASHBOARD_APP_LOCATOR } from '@kbn/deeplinks-analytics'; +import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; import { CommandSnippet } from './command_snippet'; import { DataIngestStatus } from './data_ingest_status'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { useKubernetesFlow } from './use_kubernetes_flow'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; import { type IngestionMode } from '../shared/wired_streams_ingestion_selector'; +import { usePricingFeature } from '../shared/use_pricing_feature'; +import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; +import { WIRED_ECS_DATA_VIEW_SPEC } from '../shared/wired_streams_data_view'; +import type { ObservabilityOnboardingContextValue } from '../../../plugin'; +import type { ActionLink } from './data_ingest_status'; + +const CLUSTER_OVERVIEW_DASHBOARD_ID = 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'; export const KubernetesPanel: React.FC = () => { const { data, status, error, refetch } = useKubernetesFlow(); const { onPageReady } = usePerformanceContext(); const [ingestionMode, setIngestionMode] = useState('classic'); + const metricsOnboardingEnabled = usePricingFeature( + ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING + ); + const { + services: { share }, + } = useKibana(); + const dashboardLocator = share.url.locators.get(DASHBOARD_APP_LOCATOR); + const logsLocator = share.url.locators.get(LOGS_LOCATOR_ID); + const useWiredStreams = ingestionMode === 'wired'; + const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_ECS_DATA_VIEW_SPEC } : {}; + + const [dataReceived, setDataReceived] = useState(false); + + const hasPreExistingDataEarly = usePreExistingDataCheck({ + flow: 'kubernetes', + onboardingId: data?.onboardingId, + }); - const isMonitoringStepActive = useWindowBlurDataMonitoringTrigger({ + const windowBlurred = useWindowBlurDataMonitoringTrigger({ isActive: status === FETCH_STATUS.SUCCESS, onboardingFlowType: 'kubernetes', onboardingId: data?.onboardingId, }); + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + useEffect(() => { if (data) { onPageReady({ @@ -44,6 +74,42 @@ export const KubernetesPanel: React.FC = () => { return ; } + const kubernetesActionLinks: ActionLink[] = [ + ...(metricsOnboardingEnabled + ? [ + { + id: CLUSTER_OVERVIEW_DASHBOARD_ID, + label: i18n.translate( + 'xpack.observability_onboarding.kubernetesPanel.exploreDashboard', + { defaultMessage: 'Explore Kubernetes cluster' } + ), + title: i18n.translate( + 'xpack.observability_onboarding.kubernetesPanel.monitoringCluster', + { + defaultMessage: 'Overview your Kubernetes cluster with this pre-made dashboard', + } + ), + requires: 'metrics' as const, + href: + dashboardLocator?.getRedirectUrl({ + dashboardId: CLUSTER_OVERVIEW_DASHBOARD_ID, + }) ?? '', + }, + ] + : []), + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.kubernetesPanel.logsTitle', { + defaultMessage: 'View and analyze your logs:', + }), + label: i18n.translate('xpack.observability_onboarding.kubernetesPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + requires: 'logs' as const, + href: logsLocator?.getRedirectUrl(logsLocatorParams) ?? '', + }, + ]; + const steps = [ { title: i18n.translate( @@ -82,9 +148,20 @@ export const KubernetesPanel: React.FC = () => { defaultMessage: 'Monitor your Kubernetes cluster', } ), - status: (isMonitoringStepActive ? 'current' : 'incomplete') as EuiStepStatus, + status: (dataReceived || hasPreExistingDataEarly + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, children: isMonitoringStepActive && data && ( - + setDataReceived(true)} + /> ), }, ]; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx index a19ec55977fa8..7c9b8dedea3a1 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_apm/index.tsx @@ -5,14 +5,17 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { css } from '@emotion/react'; import type { EuiBasicTableColumn } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; import { EuiBasicTable, EuiButtonIcon, + EuiFieldText, + EuiFormRow, EuiLink, EuiMarkdownFormat, EuiPanel, @@ -26,8 +29,12 @@ import { import { useKibana } from '@kbn/kibana-react-plugin/public'; import type { ValuesType } from 'utility-types'; import { usePerformanceContext } from '@kbn/ebt-tools'; -import { FETCH_STATUS } from '@kbn/observability-shared-plugin/public'; import type { ObservabilityOnboardingAppServices } from '../../..'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; +import { ProgressIndicator } from '../shared/progress_indicator'; import { useFlowBreadcrumb } from '../../shared/use_flow_breadcrumbs'; import { FeedbackButtons } from '../shared/feedback_buttons'; import { GetStartedPanel } from '../shared/get_started_panel'; @@ -35,6 +42,9 @@ import { ManagedOtlpCallout } from '../shared/managed_otlp_callout'; import { useOtelApmFlow } from './use_otel_apm_flow'; import { EmptyPrompt } from '../shared/empty_prompt'; +const FETCH_INTERVAL = 2000; +const SHOW_TROUBLESHOOTING_DELAY = 120_000; + export function OtelApmQuickstartFlow() { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.otelApm.breadcrumbs.k8sOtel', { @@ -46,6 +56,7 @@ export function OtelApmQuickstartFlow() { } = useKibana(); const { data, status, error, refetch } = useOtelApmFlow(); const { onPageReady } = usePerformanceContext(); + const [serviceName, setServiceName] = useState(''); useEffect(() => { if (data) { @@ -57,6 +68,41 @@ export function OtelApmQuickstartFlow() { } }, [data, onPageReady]); + const hasPreExistingDataEarly = usePreExistingDataCheck({ flow: 'otel_apm' }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS, + onboardingFlowType: 'otel_apm', + onboardingId: data?.onboardingId, + }); + + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + + // Set sessionStartTime when monitoring begins (first blur or early + // pre-existing data detection) rather than on mount, to narrow the + // time-window and reduce false positives from other APM services + // already ingesting data on the same cluster. + const [sessionStartTime, setSessionStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && sessionStartTime === null) { + setSessionStartTime(new Date().toISOString()); + } + }, [isMonitoringStepActive, sessionStartTime]); + + const trimmedServiceName = serviceName.trim(); + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'otel_apm', + onboardingId: data?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/otel_apm/has-data', + extraQueryParams: trimmedServiceName ? { serviceName: trimmedServiceName } : undefined, + }); + + const hasPreExistingDataFinal = hasPreExistingData || hasPreExistingDataEarly; + if (error !== undefined) { return ; } @@ -98,6 +144,8 @@ export function OtelApmQuickstartFlow() { )} @@ -107,42 +155,88 @@ export function OtelApmQuickstartFlow() { title: i18n.translate('xpack.observability_onboarding.otelApm.monitorStepTitle', { defaultMessage: 'Visualize your data', }), - children: ( + status: (hasData || hasPreExistingDataFinal + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive ? ( <> -

- -

- - + + + + {i18n.translate( + 'xpack.observability_onboarding.otelApm.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } + )} + + ), + }} + /> + + + )} + + {(hasData === true || hasPreExistingDataFinal) && ( + <> + + + id: 'services', + title: i18n.translate( + 'xpack.observability_onboarding.otelApm.servicesTitle', + { defaultMessage: 'View and analyze your services' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelApm.servicesLabel', + { defaultMessage: 'Open Service Inventory' } + ), + href: apmLocator?.getRedirectUrl({ serviceName: undefined }) ?? '', + }, + ]} + /> + + )} - ), + ) : null, }, ]} /> @@ -173,7 +267,7 @@ function InstallSDKInstructions() { {i18n.translate('xpack.observability_onboarding.otelApm.EDOTDocumentationLinkLabel', { defaultMessage: 'Elastic Distribution of OpenTelemetry documentation', @@ -208,10 +302,15 @@ function InstallSDKInstructions() { function ConfigureSDKInstructions({ managedOtlpServiceUrl, apiKeyEncoded, + serviceName, + onServiceNameChange, }: { managedOtlpServiceUrl: string; apiKeyEncoded: string; + serviceName: string; + onServiceNameChange: (value: string) => void; }) { + const serviceNameDisplay = serviceName.trim() || ''; const items = [ { setting: 'OTEL_EXPORTER_OTLP_ENDPOINT', @@ -223,8 +322,7 @@ function ConfigureSDKInstructions({ }, { setting: 'OTEL_RESOURCE_ATTRIBUTES', - value: - 'service.name=,service.version=,deployment.environment=production', + value: `service.name=${serviceNameDisplay},service.version=,deployment.environment=production`, }, ]; @@ -265,15 +363,42 @@ function ConfigureSDKInstructions({ return ( <> + + onServiceNameChange(e.target.value)} + /> + + {i18n.translate('xpack.observability_onboarding.otelApm.configureAgent.textPre', { defaultMessage: - 'Set the following variables in your application’s environment to configure the SDK:', + "Set the following variables in your application's environment to configure the SDK:", })} - + ); } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts index 6ce9eb2490e79..422373fd728d2 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.test.ts @@ -10,28 +10,27 @@ import { OTEL_KUBE_STACK_VERSION, OTEL_STACK_NAMESPACE } from './constants'; import { buildInstallStackCommand } from './build_install_stack_command'; import { buildValuesFileUrl } from './build_values_file_url'; -describe('buildValuesFileUrl()', () => { - const isMetricsOnboardingEnabled = true; - +const TEST_ONBOARDING_ID = 'test-onboarding-id'; + +const defaultArgs = { + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: false, + managedOtlpEndpointUrl: 'https://example.com/otlp', + elasticsearchUrl: 'https://example.com/elasticsearch', + apiKeyEncoded: 'encoded_api_key', + agentVersion: '9.1.0', + onboardingId: TEST_ONBOARDING_ID, +} as const; + +describe('buildInstallStackCommand()', () => { it('builds command with Elasticsearch endpoint when OTLP service is not available', () => { - const isManagedOtlpServiceAvailable = false; - const managedOtlpEndpointUrl = 'https://example.com/otlp'; - const elasticsearchUrl = 'https://example.com/elasticsearch'; - const apiKeyEncoded = 'encoded_api_key'; - const agentVersion = '9.1.0'; + const { elasticsearchUrl, apiKeyEncoded, agentVersion } = defaultArgs; const otelKubeStackValuesFileUrl = buildValuesFileUrl({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - agentVersion, - }); - const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpEndpointUrl, - elasticsearchUrl, - apiKeyEncoded, + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: false, agentVersion, }); + const command = buildInstallStackCommand(defaultArgs); expect(command).toEqual(`kubectl create namespace ${OTEL_STACK_NAMESPACE} kubectl create secret generic elastic-secret-otel \\ @@ -41,27 +40,24 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'`); + --version '${OTEL_KUBE_STACK_VERSION}' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id'`); }); it('builds command with OTLP endpoint when OTLP service is available', () => { - const isManagedOtlpServiceAvailable = true; - const managedOtlpEndpointUrl = 'https://example.com/otlp'; - const elasticsearchUrl = 'https://example.com/elasticsearch'; - const apiKeyEncoded = 'encoded_api_key'; - const agentVersion = '9.1.0'; + const { managedOtlpEndpointUrl, apiKeyEncoded, agentVersion } = defaultArgs; const otelKubeStackValuesFileUrl = buildValuesFileUrl({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, + isMetricsOnboardingEnabled: true, + isManagedOtlpServiceAvailable: true, agentVersion, }); const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled, - isManagedOtlpServiceAvailable, - managedOtlpEndpointUrl, - elasticsearchUrl, - apiKeyEncoded, - agentVersion, + ...defaultArgs, + isManagedOtlpServiceAvailable: true, }); expect(command).toEqual(`kubectl create namespace ${OTEL_STACK_NAMESPACE} @@ -72,18 +68,50 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'`); + --version '${OTEL_KUBE_STACK_VERSION}' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id'`); + }); + + describe('onboarding_id processor', () => { + it('always injects resource/onboarding_id processor into logs pipeline', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + isMetricsOnboardingEnabled: false, + }); + + expect(command).toContain('resource\\/onboarding_id'); + expect(command).toContain(`onboarding_id.attributes[0].value=${TEST_ONBOARDING_ID}`); + expect(command).toContain( + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/onboarding_id' + ); + }); + + it('injects resource/onboarding_id into metrics pipeline when metrics enabled', () => { + const command = buildInstallStackCommand(defaultArgs); + + expect(command).toContain( + 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[8]=resource/onboarding_id' + ); + }); + + it('does not inject resource/onboarding_id into metrics pipeline when metrics disabled', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + isMetricsOnboardingEnabled: false, + }); + + expect(command).not.toContain('metrics\\/node\\/otel'); + }); }); describe('wired streams', () => { it('does not include wired streams config when useWiredStreams is false', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: false, }); @@ -93,60 +121,46 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub it('routes daemon logs to wired streams when useWiredStreams is true (direct ES)', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, + ...defaultArgs, isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); expect(command).toContain('collectors.daemon.config.processors.resource\\/wired_streams'); expect(command).toContain('elasticsearch.index'); expect(command).toContain( - 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams' + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[9]=resource/wired_streams' ); expect(command).not.toContain('logs\\/apm'); }); it('routes daemon logs to wired streams when useWiredStreams is true (managed OTLP)', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, + ...defaultArgs, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); expect(command).toContain('collectors.daemon.config.processors.resource\\/wired_streams'); expect(command).toContain('elasticsearch.index'); expect(command).toContain( - 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams' + 'collectors.daemon.config.service.pipelines.logs\\/node.processors[9]=resource/wired_streams' ); expect(command).not.toContain('logs\\/apm'); }); it('excludes APM logs from wired streams regardless of metrics onboarding setting', () => { const withMetrics = buildInstallStackCommand({ + ...defaultArgs, isMetricsOnboardingEnabled: true, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); const withoutMetrics = buildInstallStackCommand({ + ...defaultArgs, isMetricsOnboardingEnabled: false, isManagedOtlpServiceAvailable: true, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', useWiredStreams: true, }); @@ -156,12 +170,7 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub it('does not modify gateway config when useWiredStreams is true', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: true, }); @@ -169,21 +178,25 @@ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kub expect(command).not.toContain('logs_index=logs'); }); - it('appends wired streams config at the end of the helm command', () => { + it('assigns onboarding_id to logs processors[8] and wired_streams to logs processors[9]', () => { const command = buildInstallStackCommand({ - isMetricsOnboardingEnabled: true, - isManagedOtlpServiceAvailable: false, - managedOtlpEndpointUrl: 'https://example.com/otlp', - elasticsearchUrl: 'https://example.com/elasticsearch', - apiKeyEncoded: 'encoded_api_key', - agentVersion: '9.1.0', + ...defaultArgs, useWiredStreams: true, }); - expect(command).toContain( - `--version '${OTEL_KUBE_STACK_VERSION}' \\ - --set 'collectors.daemon.config.processors.resource\\/wired_streams` - ); + expect(command).toContain('logs\\/node.processors[8]=resource/onboarding_id'); + expect(command).toContain('logs\\/node.processors[9]=resource/wired_streams'); + }); + + it('appends wired streams config after onboarding_id config', () => { + const command = buildInstallStackCommand({ + ...defaultArgs, + useWiredStreams: true, + }); + + const onboardingIdIndex = command.indexOf('resource/onboarding_id'); + const wiredStreamsIndex = command.indexOf('resource/wired_streams'); + expect(onboardingIdIndex).toBeLessThan(wiredStreamsIndex); }); }); }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts index 8a3bab765783c..a25ae06deb2ab 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/build_install_stack_command.ts @@ -16,6 +16,7 @@ export function buildInstallStackCommand({ apiKeyEncoded, agentVersion, useWiredStreams = false, + onboardingId, }: { isMetricsOnboardingEnabled: boolean; isManagedOtlpServiceAvailable: boolean; @@ -24,6 +25,7 @@ export function buildInstallStackCommand({ apiKeyEncoded: string; agentVersion: string; useWiredStreams?: boolean; + onboardingId: string; }): string { const ingestEndpointUrl = isManagedOtlpServiceAvailable ? managedOtlpEndpointUrl @@ -37,22 +39,33 @@ export function buildInstallStackCommand({ agentVersion, }); + // The base kube-stack Helm values file defines processors[0..7] for the + // logs/node and metrics/node/otel pipelines. Custom processors + // (onboarding_id, wired_streams) are appended starting at index 8. + let nextLogProcessorIndex = 8; + let nextMetricProcessorIndex = 8; + + const onboardingIdConfig = (() => { + let config = ` \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].action=upsert' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].key=onboarding.id' \\ + --set 'collectors.daemon.config.processors.resource\\/onboarding_id.attributes[0].value=${onboardingId}' \\ + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[${nextLogProcessorIndex++}]=resource/onboarding_id'`; + if (isMetricsOnboardingEnabled) { + config += ` \\ + --set 'collectors.daemon.config.service.pipelines.metrics\\/node\\/otel.processors[${nextMetricProcessorIndex++}]=resource/onboarding_id'`; + } + return config; + })(); + const wiredStreamsConfig = (() => { if (!useWiredStreams) return ''; - // Route container logs to wired streams by injecting the - // resource/wired_streams processor on the daemon collector's node pipeline. - // APM logs are intentionally excluded — wired streams support for APM is - // tracked separately and not yet ready. - // K8s event logs (from the cluster collector) are unaffected and keep - // their classic routing (e.g. logs-k8sobjectsreceiver.otel-default). - // The elasticsearch.index resource attribute works for both direct ES - // (elasticsearch exporter) and managed OTLP (otlp/ingest exporter). return ` \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].action=upsert' \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].key=elasticsearch.index' \\ --set 'collectors.daemon.config.processors.resource\\/wired_streams.attributes[0].value=logs.otel' \\ - --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[8]=resource/wired_streams'`; + --set 'collectors.daemon.config.service.pipelines.logs\\/node.processors[${nextLogProcessorIndex++}]=resource/wired_streams'`; })(); return `kubectl create namespace ${OTEL_STACK_NAMESPACE} @@ -63,5 +76,5 @@ kubectl create secret generic elastic-secret-otel \\ helm upgrade --install opentelemetry-kube-stack open-telemetry/opentelemetry-kube-stack \\ --namespace ${OTEL_STACK_NAMESPACE} \\ --values '${otelKubeStackValuesFileUrl}' \\ - --version '${OTEL_KUBE_STACK_VERSION}'${wiredStreamsConfig}`; + --version '${OTEL_KUBE_STACK_VERSION}'${onboardingIdConfig}${wiredStreamsConfig}`; } diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx index ac1bf69602fe4..25f8e94df1b7b 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_kubernetes/otel_kubernetes_panel.tsx @@ -6,6 +6,7 @@ */ import React, { useState, useEffect } from 'react'; +import type { EuiStepStatus } from '@elastic/eui'; import { EuiPanel, EuiSkeletonText, @@ -30,9 +31,12 @@ import { usePerformanceContext } from '@kbn/ebt-tools'; import { type LogsLocatorParams, LOGS_LOCATOR_ID } from '@kbn/logs-shared-plugin/common'; import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; import { type ObservabilityOnboardingAppServices } from '../../..'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { EmptyPrompt } from '../shared/empty_prompt'; -import { GetStartedPanel } from '../shared/get_started_panel'; import { FeedbackButtons } from '../shared/feedback_buttons'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { DataIngestStatus, type ActionLink } from '../kubernetes/data_ingest_status'; import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; import { WiredStreamsIngestionSelector, @@ -59,7 +63,7 @@ export const OtelKubernetesPanel: React.FC = () => { defaultMessage: 'Kubernetes: OpenTelemetry', }), }); - const { data, error, refetch } = useKubernetesFlow('kubernetes_otel'); + const { data, status, error, refetch } = useKubernetesFlow('kubernetes_otel'); const [idSelected, setIdSelected] = useState('nodejs'); const { services: { share, docLinks }, @@ -83,6 +87,22 @@ export const OtelKubernetesPanel: React.FC = () => { } = useWiredStreamsStatus(); const [ingestionMode, setIngestionMode] = useState('classic'); const useWiredStreams = ingestionMode === 'wired'; + + const [dataReceived, setDataReceived] = useState(false); + + const hasPreExistingDataEarly = usePreExistingDataCheck({ + flow: 'kubernetes', + onboardingId: data?.onboardingId, + enabled: useWiredStreams, + }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS, + onboardingFlowType: 'kubernetes_otel', + onboardingId: data?.onboardingId, + }); + + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; const logsLocatorParams = useWiredStreams ? { dataViewSpec: WIRED_OTEL_DATA_VIEW_SPEC } : {}; useEffect(() => { @@ -118,9 +138,56 @@ export const OtelKubernetesPanel: React.FC = () => { apiKeyEncoded: data.apiKeyEncoded, agentVersion: data.elasticAgentVersionInfo.agentBaseVersion, useWiredStreams, + onboardingId: data.onboardingId, }) : undefined; + const otelKubernetesActionLinks: ActionLink[] = [ + ...(isMetricsOnboardingEnabled + ? [ + { + id: CLUSTER_OVERVIEW_DASHBOARD_ID, + title: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster', + { defaultMessage: 'Check your Kubernetes cluster health:' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.exploreDashboard', + { defaultMessage: 'Explore Kubernetes Cluster Dashboard' } + ), + requires: 'metrics' as const, + href: + dashboardLocator?.getRedirectUrl({ + dashboardId: CLUSTER_OVERVIEW_DASHBOARD_ID, + }) ?? '', + }, + { + id: 'services', + title: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.servicesTitle', + { defaultMessage: 'Check your application services:' } + ), + label: i18n.translate( + 'xpack.observability_onboarding.otelKubernetesPanel.servicesLabel', + { defaultMessage: 'Explore Service inventory' } + ), + requires: 'metrics' as const, + href: apmLocator?.getRedirectUrl({ serviceName: undefined }) ?? '', + }, + ] + : []), + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelKubernetesPanel.logsTitle', { + defaultMessage: 'View and analyze your logs:', + }), + label: i18n.translate('xpack.observability_onboarding.otelKubernetesPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: logsLocator?.getRedirectUrl(logsLocatorParams) ?? '', + }, + ]; + return ( @@ -227,7 +294,7 @@ export const OtelKubernetesPanel: React.FC = () => { 'xpack.observability_onboarding.otelKubernetesPanel.helmsAutogeneratedTLSCertificatesTextLabel', { defaultMessage: - "Helm's autogenerated TLS certificates have a default expiration period of 365 days. These certificates are not renewed automatically unless the release is manually updated. Enabling cert-manager allows for automatic certificate renewal.", + "Helm's autogenerated TLS certificates have a default expiration period of 365 days. These certificates are not renewed automatically unless the release is manually updated. Enabling cert-manager allows for automatic certificate renewal.", } )} position="top" @@ -443,93 +510,21 @@ kubectl describe pod -n my-namespace`} defaultMessage: 'Visualize your data', } ), - children: data ? ( - <> -

- {isMetricsOnboardingEnabled && ( - - )} - {!isMetricsOnboardingEnabled && ( - - )} -

- - - - ) : ( - + status: (dataReceived || hasPreExistingDataEarly + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive && data && ( + setDataReceived(true)} + respectPreExistingData={useWiredStreams} + /> ), }, ]} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx index 2b7815c4f395d..b6e4a28bdf84e 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx @@ -18,10 +18,11 @@ import { EuiText, EuiButtonGroup, EuiCopy, - EuiImage, EuiCallOut, EuiSkeletonText, } from '@elastic/eui'; +import type { EuiStepStatus } from '@elastic/eui'; +import { css } from '@emotion/react'; import { i18n } from '@kbn/i18n'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import useAsyncFn from 'react-use/lib/useAsyncFn'; @@ -31,6 +32,11 @@ import { usePerformanceContext } from '@kbn/ebt-tools'; import { ObservabilityOnboardingPricingFeature } from '../../../../common/pricing_features'; import type { ObservabilityOnboardingAppServices } from '../../..'; import { useFetcher } from '../../../hooks/use_fetcher'; +import { usePreExistingDataCheck } from '../shared/use_pre_existing_data_check'; +import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { useTimeWindowDataDetection } from '../shared/use_time_window_data_detection'; +import { ProgressIndicator } from '../shared/progress_indicator'; +import { GetStartedPanel } from '../shared/get_started_panel'; import { useWiredStreamsStatus } from '../../../hooks/use_wired_streams_status'; import { MultiIntegrationInstallBanner } from './multi_integration_install_banner'; import { EmptyPrompt } from '../shared/empty_prompt'; @@ -54,6 +60,9 @@ const HOST_COMMAND = i18n.translate( } ); +const FETCH_INTERVAL = 2000; +const SHOW_TROUBLESHOOTING_DELAY = 120_000; + export const OtelLogsPanel: React.FC = () => { useFlowBreadcrumb({ text: i18n.translate('xpack.observability_onboarding.autoDetectPanel.breadcrumbs.otelHost', { @@ -62,7 +71,7 @@ export const OtelLogsPanel: React.FC = () => { }); const { onPageReady } = usePerformanceContext(); const { - services: { share, http, docLinks }, + services: { share, docLinks }, } = useKibana(); const { @@ -87,6 +96,38 @@ export const OtelLogsPanel: React.FC = () => { } }, [onPageReady, setupData]); + const [selectedTab, setSelectedTab] = useState('linux'); + + const hasPreExistingDataEarly = usePreExistingDataCheck({ flow: 'otel_host' }); + + const windowBlurred = useWindowBlurDataMonitoringTrigger({ + isActive: !!setupData, + onboardingFlowType: 'otel_logs', + onboardingId: setupData?.onboardingId, + }); + + const isMonitoringStepActive = windowBlurred || hasPreExistingDataEarly; + + // Set sessionStartTime when monitoring begins, not on mount. + const [sessionStartTime, setSessionStartTime] = useState(null); + useEffect(() => { + if (isMonitoringStepActive && sessionStartTime === null) { + setSessionStartTime(new Date().toISOString()); + } + }, [isMonitoringStepActive, sessionStartTime]); + + const { hasData, hasPreExistingData, isTroubleshootingVisible } = useTimeWindowDataDetection({ + isMonitoringActive: isMonitoringStepActive && sessionStartTime !== null, + sessionStartTime: sessionStartTime ?? '', + fetchInterval: FETCH_INTERVAL, + troubleshootingDelay: SHOW_TROUBLESHOOTING_DELAY, + flowType: 'otel_logs', + onboardingId: setupData?.onboardingId ?? '', + endpoint: '/internal/observability_onboarding/otel_host/has-data', + }); + + const hasPreExistingDataFinal = hasPreExistingData || hasPreExistingDataEarly; + const isMetricsOnboardingEnabled = usePricingFeature( ObservabilityOnboardingPricingFeature.METRICS_ONBOARDING ); @@ -119,6 +160,40 @@ export const OtelLogsPanel: React.FC = () => { getDeeplinks(); }, [getDeeplinks]); + const visualizeActionLinks = useMemo( + () => [ + ...(deeplinks?.logs + ? [ + { + id: 'logs', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsTitle', { + defaultMessage: 'View and analyze your logs', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.logsLabel', { + defaultMessage: 'Explore logs', + }), + href: deeplinks.logs, + }, + ] + : []), + ...(isMetricsOnboardingEnabled && deeplinks?.metrics + ? [ + { + id: 'metrics', + title: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsTitle', { + defaultMessage: 'View and analyze your metrics', + }), + label: i18n.translate('xpack.observability_onboarding.otelLogsPanel.metricsLabel', { + defaultMessage: 'Open Hosts', + }), + href: deeplinks.metrics, + }, + ] + : []), + ], + [deeplinks, isMetricsOnboardingEnabled] + ); + const installTabContents = useMemo( () => [ { @@ -182,8 +257,6 @@ export const OtelLogsPanel: React.FC = () => { [setupData, isMetricsOnboardingEnabled, isManagedOtlpServiceAvailable, useWiredStreams] ); - const [selectedTab, setSelectedTab] = React.useState(installTabContents[0].id); - const selectedContent = installTabContents.find((tab) => tab.id === selectedTab)!; if (error) { @@ -359,111 +432,79 @@ export const OtelLogsPanel: React.FC = () => { defaultMessage: 'Visualize your data', } ), - children: ( + status: (hasData || hasPreExistingDataFinal + ? 'complete' + : isMonitoringStepActive + ? 'current' + : 'incomplete') as EuiStepStatus, + children: isMonitoringStepActive ? ( <> - -

- {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel', - { - defaultMessage: - 'After running the previous command, come back and view your data.', - } - )} -

- - - - - - - - - {deeplinks?.logs && ( - <> - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel', - { defaultMessage: 'View and analyze your logs' } - )} - - - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.exploreLogs', - { - defaultMessage: 'Explore logs', - } - )} - - - - )} - - {isMetricsOnboardingEnabled && deeplinks?.metrics && ( - <> - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel', - { defaultMessage: 'View and analyze your metrics' } - )} - - - + {!(hasPreExistingDataFinal && !hasData) && ( + + )} + + {isTroubleshootingVisible && ( + <> + + + {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.exploreMetrics', - { - defaultMessage: 'Open Hosts', - } + 'xpack.observability_onboarding.otelLogsPanel.troubleshootingLinkText', + { defaultMessage: 'Open documentation' } )} - - - )} - - - - - - - {i18n.translate( - 'xpack.observability_onboarding.otelLogsPanel.documentationLink', - { - defaultMessage: 'Open documentation', - } - )} - - ), - }} - /> - + ), + }} + /> + + + )} + + {(hasData === true || hasPreExistingDataFinal) && + visualizeActionLinks.length > 0 && ( + <> + + + + )} - ), + ) : null, }, ]} /> diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx index 1a1e2da18934e..7af267a9abd73 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/get_started_panel.tsx @@ -22,6 +22,14 @@ import type { OnboardingFlowEventContext } from '../../../../common/telemetry_ev import { OBSERVABILITY_ONBOARDING_FLOW_DATASET_DETECTED_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; import type { ObservabilityOnboardingContextValue } from '../../../plugin'; +export interface ActionLink { + id: string; + title: string; + label: string; + href: string; + requires?: 'any' | 'logs' | 'metrics' | 'traces'; +} + export function GetStartedPanel({ onboardingFlowType, dataset, @@ -37,12 +45,7 @@ export function GetStartedPanel({ dataset: string; integration?: string; newTab: boolean; - actionLinks: Array<{ - id: string; - title: string; - label: string; - href: string; - }>; + actionLinks: ActionLink[]; previewImage?: string; isLoading: boolean; onboardingId?: string; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts new file mode 100644 index 0000000000000..d5b8977e453c1 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_pre_existing_data_check.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState } from 'react'; +import { useFetcher } from '../../../hooks/use_fetcher'; + +const FLOW_ENDPOINTS = { + kubernetes: '/internal/observability_onboarding/kubernetes/{onboardingId}/has-data', + otel_host: '/internal/observability_onboarding/otel_host/has-data', + otel_apm: '/internal/observability_onboarding/otel_apm/has-data', +} as const; + +type PreExistingDataFlow = keyof typeof FLOW_ENDPOINTS; + +export function usePreExistingDataCheck({ + flow, + onboardingId, + enabled = true, +}: { + flow: PreExistingDataFlow; + onboardingId?: string; + enabled?: boolean; +}): boolean { + const endpoint = FLOW_ENDPOINTS[flow]; + const needsOnboardingId = flow === 'kubernetes'; + const [start] = useState(() => new Date().toISOString()); + + const { data } = useFetcher( + (callApi): Promise<{ hasPreExistingData?: boolean }> | undefined => { + if (!enabled) return; + if (needsOnboardingId && !onboardingId) return; + return callApi(`GET ${endpoint}` as Parameters[0], { + params: { + ...(onboardingId ? { path: { onboardingId } } : {}), + query: { start }, + }, + }); + }, + [endpoint, start, onboardingId, needsOnboardingId, enabled], + { showToastOnError: false } + ); + + return data?.hasPreExistingData ?? false; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts new file mode 100644 index 0000000000000..154b6fb4a78ae --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/shared/use_time_window_data_detection.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect, useMemo, useState } from 'react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../../common/telemetry_events'; +import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import type { ObservabilityOnboardingContextValue } from '../../../plugin'; + +// After this many consecutive "no data" responses with filters applied, +// drop the extra filters and fall back to the basic time-window query. +// This prevents users from getting stuck if a field like host.os.type +// is missing from indexed documents due to mapping or collector differences. +const FALLBACK_POLL_THRESHOLD = 30; + +interface UseTimeWindowDataDetectionOptions { + isMonitoringActive: boolean; + sessionStartTime: string; + fetchInterval: number; + troubleshootingDelay: number; + flowType: string; + onboardingId: string; + endpoint: string; + extraQueryParams?: Record; +} + +export function useTimeWindowDataDetection({ + isMonitoringActive, + sessionStartTime, + fetchInterval, + troubleshootingDelay, + flowType, + onboardingId, + endpoint, + extraQueryParams, +}: UseTimeWindowDataDetectionOptions) { + const [checkDataStartTime, setCheckDataStartTime] = useState(null); + const [dataReceivedTelemetrySent, setDataReceivedTelemetrySent] = useState(false); + const [noDataPollCount, setNoDataPollCount] = useState(0); + const { + services: { analytics }, + } = useKibana(); + + useEffect(() => { + if (isMonitoringActive && checkDataStartTime === null) { + setCheckDataStartTime(Date.now()); + } + }, [isMonitoringActive, checkDataStartTime]); + + const stableExtraQueryParams = useMemo( + () => extraQueryParams, + // eslint-disable-next-line react-hooks/exhaustive-deps + [JSON.stringify(extraQueryParams)] + ); + + // After FALLBACK_POLL_THRESHOLD consecutive "no data" responses, + // drop extra filters (e.g. osType) and fall back to basic time-window. + const hasExtraParams = + stableExtraQueryParams !== undefined && Object.keys(stableExtraQueryParams).length > 0; + const shouldFallback = hasExtraParams && noDataPollCount >= FALLBACK_POLL_THRESHOLD; + const effectiveExtraParams = shouldFallback ? undefined : stableExtraQueryParams; + + const { + data: hasDataResponse, + status: hasDataStatus, + refetch: refetchHasData, + } = useFetcher( + (callApi): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> | undefined => { + if (!isMonitoringActive) return; + return callApi(`GET ${endpoint}` as Parameters[0], { + params: { + query: { start: sessionStartTime, ...effectiveExtraParams }, + }, + }); + }, + [isMonitoringActive, sessionStartTime, endpoint, effectiveExtraParams], + { showToastOnError: false } + ); + + useEffect(() => { + const pendingStatusList = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; + if ( + pendingStatusList.includes(hasDataStatus) || + hasDataResponse?.hasData === true || + hasDataResponse?.hasPreExistingData === true + ) { + return; + } + if (hasDataResponse?.hasData === false) { + setNoDataPollCount((prev) => prev + 1); + } + const timeout = setTimeout(() => { + refetchHasData(); + }, fetchInterval); + return () => clearTimeout(timeout); + }, [ + hasDataResponse?.hasData, + hasDataResponse?.hasPreExistingData, + refetchHasData, + hasDataStatus, + fetchInterval, + ]); + + useEffect(() => { + if (dataReceivedTelemetrySent) return; + + if (hasDataResponse?.hasData === true) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: flowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'complete', + }); + } else if (hasDataResponse?.hasPreExistingData === true) { + setDataReceivedTelemetrySent(true); + analytics?.reportEvent(OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT.eventType, { + flow_type: flowType, + flow_id: onboardingId, + step: 'logs-ingest', + step_status: 'pre_existing_data', + }); + } + }, [ + analytics, + hasDataResponse?.hasData, + hasDataResponse?.hasPreExistingData, + dataReceivedTelemetrySent, + flowType, + onboardingId, + ]); + + // Treat both "hasData === false" and fetch failures (where hasDataResponse + // is undefined but the request completed) as "no data yet" for the purpose + // of showing troubleshooting guidance. Without this, persistent fetch + // errors would leave the UI in a "waiting forever" state. + const noDataConfirmed = + hasDataResponse?.hasData === false || hasDataStatus === FETCH_STATUS.FAILURE; + + const isTroubleshootingVisible = + isMonitoringActive && + noDataConfirmed && + !hasDataResponse?.hasPreExistingData && + checkDataStartTime !== null && + Date.now() - checkDataStartTime > troubleshootingDelay; + + return { + hasData: hasDataResponse?.hasData ?? false, + hasPreExistingData: hasDataResponse?.hasPreExistingData ?? false, + isTroubleshootingVisible, + }; +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts new file mode 100644 index 0000000000000..6fdad24a31827 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import { checkPreExistingData } from './check_pre_existing_data'; + +const createMockEsClient = (response?: unknown, error?: Error) => { + const search = error ? jest.fn().mockRejectedValue(error) : jest.fn().mockResolvedValue(response); + return { search } as unknown as Parameters[0]; +}; + +const hitsResponse = (count: number) => ({ + hits: { total: { value: count, relation: 'eq' }, max_score: null, hits: [] }, +}); + +describe('checkPreExistingData', () => { + const indices = ['logs.otel*', 'metrics.otel*']; + const start = '2026-03-31T10:00:00.000Z'; + + it('returns true when documents exist before start', async () => { + const client = createMockEsClient(hitsResponse(5)); + expect(await checkPreExistingData(client, indices, start)).toBe(true); + }); + + it('returns false when no documents exist before start', async () => { + const client = createMockEsClient(hitsResponse(0)); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); + + it('queries the 5-minute window before start', async () => { + const client = createMockEsClient(hitsResponse(0)); + await checkPreExistingData(client, indices, start); + + expect(client.search).toHaveBeenCalledWith( + expect.objectContaining({ + index: indices, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: '2026-03-31T09:55:00.000Z', + lt: start, + }, + }, + }, + ], + }, + }, + }) + ); + }); + + it('returns false on no shards available error', async () => { + const error = new errors.ResponseError({ + statusCode: 503, + body: { + error: { + type: 'search_phase_execution_exception', + root_cause: [{ type: 'no_shard_available_action_exception' }], + }, + }, + headers: {}, + warnings: [], + meta: {} as never, + }); + const client = createMockEsClient(undefined, error); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); + + it('returns false on unexpected errors', async () => { + const client = createMockEsClient(undefined, new Error('Connection refused')); + expect(await checkPreExistingData(client, indices, start)).toBe(false); + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts new file mode 100644 index 0000000000000..f86166c63a046 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/check_pre_existing_data.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { estypes } from '@elastic/elasticsearch'; + +const PRE_CHECK_WINDOW_MS = 300_000; + +/** + * Checks whether data was actively flowing into the given indices + * in the 5 minutes before `start`. If it was, time-range-based + * has-data detection is likely to produce false positives. + * + * Returns `false` on any error so it never blocks the main flow. + */ +export const checkPreExistingData = async ( + esClient: ElasticsearchClient, + indices: string[], + start: string +): Promise => { + try { + const startMs = new Date(start).getTime(); + const windowStart = new Date(startMs - PRE_CHECK_WINDOW_MS).toISOString(); + + const result = await esClient.search({ + index: indices, + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: windowStart, lt: start } } }], + }, + }, + }); + + return (result.hits.total as estypes.SearchTotalHits).value > 0; + } catch { + return false; + } +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts new file mode 100644 index 0000000000000..5f91d015de628 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/lib/handle_has_data_search_error.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import { errors } from '@elastic/elasticsearch'; + +/** + * Handles errors from has-data Elasticsearch search requests. + * Returns true if the error is a transient "no shards available" condition + * (which should be treated as hasData: false), or throws a Boom error otherwise. + */ +export function isNoShardsAvailableError(error: unknown): boolean { + if (!(error instanceof errors.ResponseError)) { + return false; + } + + const body = error.body as + | { error?: { type?: string; root_cause?: Array<{ type?: string }> } } + | undefined; + const errorType = body?.error?.type; + const rootCauseType = body?.error?.root_cause?.[0]?.type; + + if ( + errorType === 'search_phase_execution_exception' && + rootCauseType === 'no_shard_available_action_exception' + ) { + return true; + } + + return false; +} + +export function throwHasDataSearchError(error: unknown): never { + const message = error instanceof Error ? error.message : String(error); + throw Boom.internal(`Elasticsearch responded with an error. ${message}`); +} diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts index 494b56051201a..c72eec6dfd0b5 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/cloudforwarder/route.ts @@ -6,11 +6,19 @@ */ import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed_otlp_service_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getManagedOtlpServiceUrl } from '../../lib/get_managed_otlp_service_url'; +import { CLOUDFORWARDER_INDEX_PATTERNS } from '../../../common/aws_cloudforwarder'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; export interface CreateCloudForwarderOnboardingFlowRouteResponse { onboardingId: string; @@ -58,6 +66,56 @@ const createCloudForwarderOnboardingFlowRoute = createObservabilityOnboardingSer }, }); +const hasCloudForwarderDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/cloudforwarder/has-data', + params: t.type({ + query: t.type({ + logType: t.keyof({ vpcflow: null, elbaccess: null, cloudtrail: null }), + start: t.string, + }), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { + const { logType, start } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + const indexPattern = CLOUDFORWARDER_INDEX_PATTERNS[logType]; + + try { + const [preExisting, result] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, [indexPattern], start), + elasticsearch.client.asCurrentUser.search({ + index: [indexPattern], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: start } } }], + }, + }, + }), + ]); + + const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; + return { hasData, hasPreExistingData: preExisting || undefined }; + } catch (error) { + if (isNoShardsAvailableError(error)) { + return { hasData: false }; + } + + throwHasDataSearchError(error); + } + }, +}); + export const cloudforwarderOnboardingRouteRepository = { ...createCloudForwarderOnboardingFlowRoute, + ...hasCloudForwarderDataRoute, }; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts new file mode 100644 index 0000000000000..e3c60986e4c52 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import type { estypes } from '@elastic/elasticsearch'; +import { resolveProbe } from './resolve_has_data_probes'; + +const fulfilledWithHits = (count: number): PromiseSettledResult => ({ + status: 'fulfilled', + value: { + hits: { total: { value: count, relation: 'eq' }, max_score: null, hits: [] }, + } as unknown as estypes.SearchResponse, +}); + +const rejectedWith = (error: Error): PromiseSettledResult => ({ + status: 'rejected', + reason: error, +}); + +const noShardsError = () => { + const error = new errors.ResponseError({ + statusCode: 503, + body: { + error: { + type: 'search_phase_execution_exception', + root_cause: [{ type: 'no_shard_available_action_exception' }], + }, + }, + headers: {}, + warnings: [], + meta: {} as never, + }); + return error; +}; + +describe('resolveProbe', () => { + it('returns true when documents are found', () => { + expect(resolveProbe(fulfilledWithHits(1))).toBe(true); + }); + + it('returns false when no documents are found', () => { + expect(resolveProbe(fulfilledWithHits(0))).toBe(false); + }); + + it('returns false on no shards available error', () => { + expect(resolveProbe(rejectedWith(noShardsError()))).toBe(false); + }); + + it('throws on unexpected errors', () => { + expect(() => resolveProbe(rejectedWith(new Error('Request timed out')))).toThrow( + 'Elasticsearch responded with an error. Request timed out' + ); + }); + + describe('combined classic + wired stream probes', () => { + it('returns true when classic probe finds data and wired probe finds nothing', () => { + const hasLogs = resolveProbe(fulfilledWithHits(1)) || resolveProbe(fulfilledWithHits(0)); + expect(hasLogs).toBe(true); + }); + + it('returns true when classic probe finds nothing but wired probe finds data', () => { + const hasLogs = resolveProbe(fulfilledWithHits(0)) || resolveProbe(fulfilledWithHits(1)); + expect(hasLogs).toBe(true); + }); + + it('returns false when both probes find nothing', () => { + const hasLogs = resolveProbe(fulfilledWithHits(0)) || resolveProbe(fulfilledWithHits(0)); + expect(hasLogs).toBe(false); + }); + + it('handles optional wired probe (undefined when no start time)', () => { + const wiredResult: PromiseSettledResult | undefined = undefined; + const hasLogs = + resolveProbe(fulfilledWithHits(0)) || (wiredResult ? resolveProbe(wiredResult) : false); + expect(hasLogs).toBe(false); + }); + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts new file mode 100644 index 0000000000000..bfaef76352799 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/resolve_has_data_probes.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { estypes } from '@elastic/elasticsearch'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; + +/** + * Resolves a has-data probe result. Returns true if documents were found, + * false for no-shards-available errors, and throws on unexpected errors. + */ +export const resolveProbe = (result: PromiseSettledResult): boolean => { + if (result.status === 'fulfilled') { + return (result.value.hits.total as estypes.SearchTotalHits).value > 0; + } + if (isNoShardsAvailableError(result.reason)) { + return false; + } + throwHasDataSearchError(result.reason); +}; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts index ca969ff1b97bb..7305c4eef9698 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/kubernetes/route.ts @@ -10,6 +10,12 @@ import * as t from 'io-ts'; import Boom from '@hapi/boom'; import { termQuery } from '@kbn/observability-plugin/server'; import type { estypes } from '@elastic/elasticsearch'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; +import { resolveProbe } from './resolve_has_data_probes'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; @@ -31,6 +37,9 @@ export interface CreateKubernetesOnboardingFlowRouteResponse { export interface HasKubernetesDataRouteResponse { hasData: boolean; + hasLogs?: boolean; + hasMetrics?: boolean; + hasPreExistingData?: boolean; } const createKubernetesOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ @@ -120,6 +129,9 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ path: t.type({ onboardingId: t.string, }), + query: t.partial({ + start: t.string, + }), }), security: { authz: { @@ -129,40 +141,107 @@ const hasKubernetesDataRoute = createObservabilityOnboardingServerRoute({ }, async handler(resources): Promise { const { onboardingId } = resources.params.path; + const { start } = resources.params.query; const { elasticsearch } = await resources.context.core; try { - const result = await elasticsearch.client.asCurrentUser.search({ - index: ['logs-*', 'metrics-*', 'logs', 'logs.*'], + const commonSearchParams = { ignore_unavailable: true, allow_partial_search_results: true, - size: 0, + size: 0 as const, terminate_after: 1, - query: { - bool: { - filter: termQuery('fields.onboarding_id', onboardingId), - }, + }; + + // Classic data streams: use indexed onboarding ID fields (fast inverted-index lookups). + const indexedQuery: estypes.QueryDslQueryContainer = { + bool: { + filter: [ + { + bool: { + should: [ + ...termQuery('fields.onboarding_id', onboardingId), + ...termQuery('resource.attributes.onboarding.id', onboardingId), + ...termQuery('labels.onboarding_id', onboardingId), + ], + minimum_should_match: 1, + }, + }, + ], }, - }); - const { value } = result.hits.total as estypes.SearchTotalHits; + }; + + const wiredStreamIndices = ['logs.otel*', 'logs.ecs*', 'metrics.otel*', 'metrics.ecs*']; + + // Check if data was already flowing into wired stream indices before + // the user started onboarding. If so, time-range detection on those + // indices would produce false positives, so we skip it. + const hasPreExistingData = start + ? await checkPreExistingData(elasticsearch.client.asCurrentUser, wiredStreamIndices, start) + : false; + + // Wired streams (logs.otel*, logs.ecs*) use passthrough mapping where + // onboarding.id is not indexed, so we cannot filter by it without a + // runtime mapping (which times out on large clusters). Instead, fall + // back to a time-range-only query when a start time is provided and + // no pre-existing data would cause false positives. + const wiredStreamQuery: estypes.QueryDslQueryContainer | undefined = + start && !hasPreExistingData + ? { bool: { filter: [{ range: { '@timestamp': { gte: start } } }] } } + : undefined; + + const searches: Array> = [ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*'], + ...commonSearchParams, + query: indexedQuery, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*'], + ...commonSearchParams, + query: indexedQuery, + }), + ]; + + if (wiredStreamQuery) { + searches.push( + elasticsearch.client.asCurrentUser.search({ + index: ['logs.otel*', 'logs.ecs*'], + ...commonSearchParams, + query: wiredStreamQuery, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics.otel*', 'metrics.ecs*'], + ...commonSearchParams, + query: wiredStreamQuery, + }) + ); + } + + const results = await Promise.allSettled(searches); + const [logsResult, metricsResult, wiredLogsResult, wiredMetricsResult] = results; + + const hasLogs = + resolveProbe(logsResult) || (wiredLogsResult ? resolveProbe(wiredLogsResult) : false); + const hasMetrics = + resolveProbe(metricsResult) || + (wiredMetricsResult ? resolveProbe(wiredMetricsResult) : false); return { - hasData: value > 0, + hasData: hasLogs || hasMetrics, + hasLogs, + hasMetrics, + hasPreExistingData: hasPreExistingData || undefined, }; } catch (error) { - const errorType = error?.meta?.body?.error?.type; - const rootCauseType = error?.meta?.body?.error?.root_cause?.[0]?.type; - - if ( - errorType === 'search_phase_execution_exception' && - rootCauseType === 'no_shard_available_action_exception' - ) { + if (isNoShardsAvailableError(error)) { return { hasData: false, + hasLogs: false, + hasMetrics: false, }; } - throw Boom.internal(`Elasticsearch responses with an error. ${error.message}`); + throwHasDataSearchError(error); } }, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts index fef013c034a5a..3c60a08466a30 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_apm/route.ts @@ -6,11 +6,18 @@ */ import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import { createManagedOtlpServiceApiKey } from '../../lib/api_key/create_managed_otlp_service_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getManagedOtlpServiceUrl } from '../../lib/get_managed_otlp_service_url'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; const createOtelApmOnboardingFlowRoute = createObservabilityOnboardingServerRoute({ endpoint: 'POST /internal/observability_onboarding/otel_apm/flow', @@ -51,6 +58,67 @@ const createOtelApmOnboardingFlowRoute = createObservabilityOnboardingServerRout }, }); +const hasOtelApmDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/otel_apm/has-data', + params: t.type({ + query: t.intersection([t.type({ start: t.string }), t.partial({ serviceName: t.string })]), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { + const { start, serviceName } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + const apmIndices = [ + 'traces-apm*', + 'traces-*.otel-*', + 'logs-apm*', + 'logs-*.otel-*', + 'metrics-apm*', + 'metrics-*.otel-*', + 'apm-*', + ]; + + try { + const filters: estypes.QueryDslQueryContainer[] = [ + { range: { '@timestamp': { gte: start } } }, + ]; + if (serviceName) { + filters.push({ term: { 'service.name': serviceName } }); + } + const query: estypes.QueryDslQueryContainer = { + bool: { filter: filters }, + }; + + const [preExisting, result] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, apmIndices, start), + elasticsearch.client.asCurrentUser.search({ + index: apmIndices, + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]); + + const hasData = (result.hits.total as estypes.SearchTotalHits).value > 0; + return { hasData, hasPreExistingData: preExisting || undefined }; + } catch (error) { + if (isNoShardsAvailableError(error)) { + return { hasData: false }; + } + + throwHasDataSearchError(error); + } + }, +}); + export const otelApmOnboardingRouteRepository = { ...createOtelApmOnboardingFlowRoute, + ...hasOtelApmDataRoute, }; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts index f21efb35146d8..a65509346ec61 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/otel_host/route.ts @@ -5,10 +5,18 @@ * 2.0. */ +import { v4 as uuidv4 } from 'uuid'; +import * as t from 'io-ts'; import Boom from '@hapi/boom'; +import type { estypes } from '@elastic/elasticsearch'; import type { ElasticAgentVersionInfo } from '../../../common/types'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; import { getFallbackESUrl } from '../../lib/get_fallback_urls'; +import { + isNoShardsAvailableError, + throwHasDataSearchError, +} from '../../lib/handle_has_data_search_error'; +import { checkPreExistingData } from '../../lib/check_pre_existing_data'; import { getAgentVersionInfo } from '../../lib/get_agent_version'; import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key'; import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges'; @@ -25,6 +33,7 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ }, }, async handler(resources): Promise<{ + onboardingId: string; elasticAgentVersionInfo: ElasticAgentVersionInfo; elasticsearchUrl: string; apiKeyEncoded: string; @@ -65,6 +74,7 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ : await createShipperApiKey(client.asCurrentUser, `otel-host`); return { + onboardingId: uuidv4(), elasticsearchUrl: elasticsearchUrlList.length > 0 ? elasticsearchUrlList[0] : '', elasticAgentVersionInfo, apiKeyEncoded, @@ -73,6 +83,74 @@ const setupFlowRoute = createObservabilityOnboardingServerRoute({ }, }); +const hasOtelHostDataRoute = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /internal/observability_onboarding/otel_host/has-data', + params: t.type({ + query: t.intersection([t.type({ start: t.string }), t.partial({ osType: t.string })]), + }), + security: { + authz: { + enabled: false, + reason: 'Authorization is checked by Elasticsearch', + }, + }, + async handler(resources): Promise<{ hasData: boolean; hasPreExistingData?: boolean }> { + const { start, osType } = resources.params.query; + const { elasticsearch } = await resources.context.core; + + const allIndices = ['logs-*.otel-*', 'logs.otel', 'logs.otel.*', 'metrics-*.otel-*']; + + const filters: estypes.QueryDslQueryContainer[] = [{ range: { '@timestamp': { gte: start } } }]; + if (osType) { + filters.push({ term: { 'host.os.type': osType } }); + } + const query: estypes.QueryDslQueryContainer = { + bool: { filter: filters }, + }; + + const [preExisting, [logsResult, metricsResult]] = await Promise.all([ + checkPreExistingData(elasticsearch.client.asCurrentUser, allIndices, start), + Promise.allSettled([ + elasticsearch.client.asCurrentUser.search({ + index: ['logs-*.otel-*', 'logs.otel', 'logs.otel.*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + elasticsearch.client.asCurrentUser.search({ + index: ['metrics-*.otel-*'], + ignore_unavailable: true, + allow_partial_search_results: true, + size: 0, + terminate_after: 1, + query, + }), + ]), + ]); + + const resolveProbe = (result: PromiseSettledResult): boolean => { + if (result.status === 'fulfilled') { + return (result.value.hits.total as estypes.SearchTotalHits).value > 0; + } + if (isNoShardsAvailableError(result.reason)) { + return false; + } + throwHasDataSearchError(result.reason); + }; + + const hasLogs = resolveProbe(logsResult); + const hasMetrics = resolveProbe(metricsResult); + + return { + hasData: hasLogs || hasMetrics, + hasPreExistingData: preExisting || undefined, + }; + }, +}); + export const otelHostOnboardingRouteRepository = { ...setupFlowRoute, + ...hasOtelHostDataRoute, }; diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json b/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json index 41397a0abbb84..33fb2822d72a8 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability_onboarding/tsconfig.json @@ -55,7 +55,8 @@ "@kbn/streams-plugin", "@kbn/data-views-plugin", "@kbn/ingest-hub-plugin", - "@kbn/shared-ux-utility" + "@kbn/shared-ux-utility", + "@kbn/core-elasticsearch-server" ], "exclude": ["target/**/*"] }