[Observability Onboarding] Add data detection & loading indicators to onboarding flows

x-pack/platform/plugins/private/translations/translations/de-DE.json

@@ -29032,7 +29032,6 @@
     "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumentieren Sie Ihre Anwendung (optional)",
     "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Überprüfen Sie die Gesundheit Ihres Kubernetes-Clusters:",
     "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Daten visualisieren",
-    "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysieren Sie die Gesundheit Ihres Kubernetes-Clusters und überwachen Sie Ihre Container-Workloads.",
     "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "Weitere Informationen finden Sie in der Dokumentation",
     "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Wählen Sie eine Programmiersprache aus.",
     "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Erkunden Sie das Service-Inventar",
@@ -29043,9 +29042,6 @@
     "xpack.observability_onboarding.otelLogs.status.failed": "Fehler bei der Installation der Integration",
     "xpack.observability_onboarding.otelLogs.status.failedDetails": "Eingehende Daten könnten möglicherweise nicht korrekt indiziert werden. Details:",
     "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Plattform wählen",
-    "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Dokumentation öffnen",
-    "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Logs durchsuchen",
-    "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Offene Hosts",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Ab dem Setup werden neue log Meldungen gesammelt.",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Der Standardlogpfad ist /var/log/*. Sie können diesen Pfad bei Bedarf in der Datei otel.yml ändern.",
     "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informationen zur Konfiguration",
@@ -29056,10 +29052,6 @@
     "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Daten visualisieren",
     "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.label": "Technische Vorschau",
     "xpack.observability_onboarding.otelLogsPanel.techPreviewBadge.tooltip": "Diese Funktionalität befindet sich in der technischen Vorschau und kann in einer zukünftigen Version geändert oder vollständig entfernt werden. Elastic wird sich bemühen, alle Probleme zu beheben, aber die Features in der technischen Vorschau unterliegen nicht dem Support-SLA der offiziellen GA-Features.",
-    "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Weitere Einzelheiten und Lösungen zur Fehlerbehebung finden Sie in unserer Dokumentation. {link}",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Ihre Metriken anzeigen und analysieren",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Ihre Logs ansehen und analysieren",
-    "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Kehren Sie nach der Ausführung des vorherigen Befehls zurück und sehen Sie sich Ihre Daten an.",
     "xpack.observability_onboarding.otelTile.description": "Erfassen Sie Protokolle und Host-Metriken mit der Elastic-Distribution des OpenTelemetry Collector",
     "xpack.observability_onboarding.otelTile.title": "OpenTelemetry",
     "xpack.observability_onboarding.packageList.uploadFileDescription": "Laden Sie Daten aus einer CSV-, TSV-, JSON- oder anderen Log-Datei zur Analyse in Elasticsearch hoch.",

x-pack/platform/plugins/private/translations/translations/fr-FR.json

@@ -29381,7 +29381,6 @@
     "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "Instrumenter votre application (facultatif)",
     "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Vérifiez l'intégrité de votre cluster Kubernetes :",
     "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "Visualiser vos données",
-    "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Analysez l'intégrité de votre cluster Kubernetes et monitorez les charges de travail de vos conteneurs.",
     "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "se reporter à la documentation",
     "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "Sélectionner un langage de programmation",
     "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "Explorer l'inventaire des services",
@@ -29392,9 +29391,6 @@
     "xpack.observability_onboarding.otelLogs.status.failed": "Échec de l'installation de l'intégration",
     "xpack.observability_onboarding.otelLogs.status.failedDetails": "Les données entrantes peuvent ne pas être indexées correctement. Détails :",
     "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "Choisissez une plateforme",
-    "xpack.observability_onboarding.otelLogsPanel.documentationLink": "Ouvrir la documentation",
-    "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "Explorer les logs",
-    "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "Ouvrir les hôtes",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "Les nouveaux messages de log sont collectés à partir de la configuration.",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "Le chemin des logs par défaut est /var/log/*. Vous pouvez si nécessaire modifier ce chemin dans le fichier otel.yml.",
     "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "Informations sur la configuration",
@@ -29403,10 +29399,6 @@
     "xpack.observability_onboarding.otelLogsPanel.steps.platform": "Sélectionnez votre plateforme",
     "xpack.observability_onboarding.otelLogsPanel.steps.start": "Lancez le collecteur",
     "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "Visualiser vos données",
-    "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "Vous trouverez plus d'informations et une solution de résolution des problèmes dans notre documentation. {link}",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "Visualisez et analysez vos métriques",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "Visualisez et analysez vos logs",
-    "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "Après avoir exécuté la commande précédente, revenez et visualisez vos données.",
     "xpack.observability_onboarding.otelTile.description": "Collectez les logs et les indicateurs de l'hôte à l'aide de la distribution Elastic du collecteur OpenTelemetry",
     "xpack.observability_onboarding.otelTile.title": "OpenTelemetry",
     "xpack.observability_onboarding.packageList.uploadFileDescription": "Téléchargez les données d'un fichier CSV, TSV, JSON ou autre fichier log vers Elasticsearch pour analyse.",

x-pack/platform/plugins/private/translations/translations/ja-JP.json

@@ -29435,7 +29435,6 @@
     "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "アプリケーションのインストルメンテーション（任意）",
     "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "Kubernetesクラスター正常性を確認：",
     "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "データを可視化する",
-    "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "Kubernetesクラスターの正常性を分析し、コンテナーのワークロードを監視します。",
     "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "ドキュメントを参照",
     "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "プログラミング言語を選択",
     "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "サービスインベントリを探索",
@@ -29446,9 +29445,6 @@
     "xpack.observability_onboarding.otelLogs.status.failed": "統合のインストールに失敗しました",
     "xpack.observability_onboarding.otelLogs.status.failedDetails": "受信データは正しくインデックス化されていない可能性があります。詳細：",
     "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "プラットフォームを選択",
-    "xpack.observability_onboarding.otelLogsPanel.documentationLink": "ドキュメントを開く",
-    "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "ログを探索",
-    "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "ホストを開く",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "今後、新しいログメッセージはセットアップから収集されます。",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "デフォルトのログのパスは/var/log/*です。必要に応じて、otel.ymlファイルでこのパスを変更できます。",
     "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "構成情報",
@@ -29457,10 +29453,6 @@
     "xpack.observability_onboarding.otelLogsPanel.steps.platform": "プラットフォームを選択",
     "xpack.observability_onboarding.otelLogsPanel.steps.start": "コレクターを開始",
     "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "データを可視化する",
-    "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "詳細とトラブルシューティングの解決策については、ドキュメントをご覧ください。{link}",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "メトリックを表示して分析",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "ログを表示して分析",
-    "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "前のコマンドを実行した後、データに戻って表示します。",
     "xpack.observability_onboarding.otelTile.description": "OpenTelemetryコレクターのElasticディストリビューションを使用して、ログとホストメトリックを収集します。",
     "xpack.observability_onboarding.otelTile.title": "OpenTelemetry",
     "xpack.observability_onboarding.packageList.uploadFileDescription": "分析するため、CSV、TSV、JSON、他のログファイルからElasticsearchにアップロードします。",

x-pack/platform/plugins/private/translations/translations/zh-CN.json

@@ -29414,7 +29414,6 @@
     "xpack.observability_onboarding.otelKubernetesPanel.instrumentApplicationStepTitle": "检测您的应用程序（可选）",
     "xpack.observability_onboarding.otelKubernetesPanel.monitoringCluster": "检查 Kubernetes 集群的运行状况：",
     "xpack.observability_onboarding.otelKubernetesPanel.monitorStepTitle": "可视化数据",
-    "xpack.observability_onboarding.otelKubernetesPanel.onceYourKubernetesInfrastructureLabel": "分析 Kubernetes 集群的运行状况并监测容器工作负载。",
     "xpack.observability_onboarding.otelKubernetesPanel.referToTheDocumentationLinkLabel": "参阅文档",
     "xpack.observability_onboarding.otelKubernetesPanel.selectProgrammingLanguageLegend": "选择编程语言",
     "xpack.observability_onboarding.otelKubernetesPanel.servicesLabel": "浏览服务库存",
@@ -29425,9 +29424,6 @@
     "xpack.observability_onboarding.otelLogs.status.failed": "集成安装失败",
     "xpack.observability_onboarding.otelLogs.status.failedDetails": "传入数据可能未正确索引。详情：",
     "xpack.observability_onboarding.otelLogsPanel.choosePlatform": "选择平台",
-    "xpack.observability_onboarding.otelLogsPanel.documentationLink": "打开文档",
-    "xpack.observability_onboarding.otelLogsPanel.exploreLogs": "浏览日志",
-    "xpack.observability_onboarding.otelLogsPanel.exploreMetrics": "打开主机",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription": "将从设置完成后收集新的日志消息。",
     "xpack.observability_onboarding.otelLogsPanel.historicalDataDescription2": "默认日志路径为 /var/log/*。如果需要，可以在 otel.yml 文件中更改此路径。",
     "xpack.observability_onboarding.otelLogsPanel.limitationTitle": "配置信息",
@@ -29436,10 +29432,6 @@
     "xpack.observability_onboarding.otelLogsPanel.steps.platform": "选择平台",
     "xpack.observability_onboarding.otelLogsPanel.steps.start": "启动收集器",
     "xpack.observability_onboarding.otelLogsPanel.steps.visualize": "可视化数据",
-    "xpack.observability_onboarding.otelLogsPanel.troubleshooting": "在我们的文档中查找更多详情和故障排除解决方案。{link}",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourMetricsTextLabel": "查看并分析您的指标",
-    "xpack.observability_onboarding.otelLogsPanel.viewAndAnalyzeYourTextLabel": "查看并分析您的日志",
-    "xpack.observability_onboarding.otelLogsPanel.waitForTheDataLabel": "运行上一个命令后，返回并查看您的数据。",
     "xpack.observability_onboarding.otelTile.description": "使用 OpenTelemetry 收集器的 Elastic 发行版收集日志和主机指标",
     "xpack.observability_onboarding.otelTile.title": "OpenTelemetry",
     "xpack.observability_onboarding.packageList.uploadFileDescription": "从 CSV、TSV、JSON 或其他日志文件上传数据到 Elasticsearch 以进行分析。",

x-pack/solutions/observability/plugins/observability_onboarding/common/aws_cloudforwarder.ts

@@ -27,3 +27,9 @@ export const CLOUDFORMATION_STACK_CONFIGS = {
 } as const;
 
 export type LogType = keyof typeof CLOUDFORMATION_STACK_CONFIGS;
+
+export const CLOUDFORWARDER_INDEX_PATTERNS: Record<LogType, string> = {
+  vpcflow: 'logs-aws.vpcflow.otel-*',
+  elbaccess: 'logs-aws.elbaccess.otel-*',
+  cloudtrail: 'logs-aws.cloudtrail.otel-*',
+};

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/host_otel.spec.ts

@@ -57,12 +57,23 @@ test('Otel Host', async ({
   fs.writeFileSync(outputPath, codeSnippet);
 
   /**
-   * There is no explicit data ingest indication
-   * in the flow, so we need to rely on a timeout.
-   * 3 minutes should be enough for the collector
-   * to initialize and start ingesting data.
+   * The page waits for the browser window to lose
+   * focus as a signal to start checking for incoming data
    */
-  await page.waitForTimeout(3 * 60000);
+  await page.evaluate('window.dispatchEvent(new Event("blur"))');
+
+  /**
+   * Wait for the data received indicator to appear.
+   * The flow polls for data after the blur event and
+   * shows "We are monitoring your host" once data arrives.
+   */
+  await otelHostFlowPage.assertDataReceivedIndicator();
+
+  /**
+   * Additional buffer to ensure data has propagated
+   * to dashboards and Discover before navigating.
+   */
+  await page.waitForTimeout(2 * 60000);
 
   /**
    * Wired streams only reroutes logs (to logs.otel); metrics and traces are

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/kubernetes_otel.spec.ts

@@ -89,12 +89,24 @@ test('Otel Kubernetes', async ({
   fs.writeFileSync(outputPath, codeSnippet);
 
   /**
-   * There is no explicit data ingest indication
-   * in the flow, so we need to rely on a timeout.
-   * 5 minutes should be enough for the stack to be
-   * created and to start pushing data.
+   * The page waits for the browser window to lose
+   * focus as a signal to start checking for incoming data
    */
-  await page.waitForTimeout(5 * 60000);
+  await page.evaluate('window.dispatchEvent(new Event("blur"))');
+
+  /**
+   * Wait for the data received indicator to appear.
+   * The flow now uses DataIngestStatus which polls for data
+   * after the blur event and shows "We are monitoring your cluster"
+   * once both logs and metrics have arrived.
+   */
+  await otelKubernetesFlowPage.assertDataReceivedIndicator();
+
+  /**
+   * Additional buffer to ensure data has propagated
+   * to dashboards and Discover before navigating.
+   */
+  await page.waitForTimeout(2 * 60000);
 
   /**
    * Wired streams only reroutes logs (to logs.otel); metrics and traces are

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_host_flow.page.ts

@@ -12,12 +12,20 @@ export class OtelHostFlowPage {
 
   private readonly exploreLogsButton: Locator;
   private readonly exploreMetricsButton: Locator;
+  private readonly dataReceivedIndicator: Locator;
 
   constructor(page: Page) {
     this.page = page;
 
-    this.exploreLogsButton = this.page.getByTestId('obltOnboardingExploreLogs');
-    this.exploreMetricsButton = this.page.getByTestId('obltOnboardingExploreMetrics');
+    this.exploreLogsButton = this.page.getByTestId(
+      'observabilityOnboardingDataIngestStatusActionLink-logs'
+    );
+    this.exploreMetricsButton = this.page.getByTestId(
+      'observabilityOnboardingDataIngestStatusActionLink-metrics'
+    );
+    this.dataReceivedIndicator = this.page
+      .getByTestId('observabilityOnboardingOtelHostDataProgressIndicator')
+      .getByText('We are monitoring your host');
   }
 
   public async selectPlatform(osName: string) {
@@ -56,6 +64,13 @@ export class OtelHostFlowPage {
     await this.exploreLogsButton.click();
   }
 
+  public async assertDataReceivedIndicator(): Promise<void> {
+    await expect(
+      this.dataReceivedIndicator,
+      'Data received indicator should be visible'
+    ).toBeVisible({ timeout: 5 * 60_000 });
+  }
+
   public async assertLogsExplorationButtonVisible() {
     await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible();
   }

x-pack/solutions/observability/plugins/observability_onboarding/e2e/playwright/stateful/pom/pages/otel_kubernetes_flow.page.ts

@@ -12,12 +12,16 @@ export class OtelKubernetesFlowPage {
   context: BrowserContext;
 
   private readonly exploreLogsButton: Locator;
+  private readonly dataReceivedIndicator: Locator;
 
   constructor(page: Page, context: BrowserContext) {
     this.page = page;
     this.context = context;
 
     this.exploreLogsButton = this.page.getByText('Explore logs');
+    this.dataReceivedIndicator = this.page
+      .getByTestId('observabilityOnboardingKubernetesPanelDataProgressIndicator')
+      .getByText('We are monitoring your cluster');
   }
 
   public async copyHelmRepositorySnippetToClipboard() {
@@ -82,6 +86,13 @@ export class OtelKubernetesFlowPage {
     }
   }
 
+  public async assertDataReceivedIndicator(): Promise<void> {
+    await expect(
+      this.dataReceivedIndicator,
+      'Data received indicator should be visible'
+    ).toBeVisible({ timeout: 5 * 60_000 });
+  }
+
   public async assertLogsExplorationButtonVisible() {
     await expect(this.exploreLogsButton, 'Logs exploration button should be visible').toBeVisible();
   }

x-pack/solutions/observability/plugins/observability_onboarding/moon.yml

@@ -60,6 +60,7 @@ dependsOn:
   - '@kbn/data-views-plugin'
   - '@kbn/ingest-hub-plugin'
   - '@kbn/shared-ux-utility'
+  - '@kbn/core-elasticsearch-server'
 tags:
   - plugin
   - prod