[kbn/es-archiver] add 'dataOnly' arg to avoid kbnClient use

src/platform/packages/shared/kbn-es-archiver/src/actions/load.test.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isSavedObjectIndex } from '../lib/indices/kibana_index';
+
+describe('loadAction - dataOnly validation', () => {
+  describe('isSavedObjectIndex', () => {
+    it('should identify .kibana saved object indices', () => {
+      expect(isSavedObjectIndex('.kibana_8.17.0_001')).toBe(true);
+      expect(isSavedObjectIndex('.kibana_task_manager_8.17.0_001')).toBe(true);
+      expect(isSavedObjectIndex('.kibana')).toBe(true);
+      expect(isSavedObjectIndex('.kibana_1')).toBe(true);
+    });
+
+    it('should not flag non-saved-object indices', () => {
+      expect(isSavedObjectIndex('logs-test')).toBe(false);
+      expect(isSavedObjectIndex('metrics-test')).toBe(false);
+      expect(isSavedObjectIndex('.kibana_security_session_1')).toBe(false);
+      expect(isSavedObjectIndex(undefined)).toBe(false);
+    });
+  });
+});

src/platform/packages/shared/kbn-es-archiver/src/actions/load.ts

@@ -27,6 +27,7 @@ import {
   createCreateIndexStream,
   createIndexDocRecordsStream,
   migrateSavedObjectIndices,
+  isSavedObjectIndex,
   Progress,
   createDefaultSpace,
   type LoadActionPerfOptions,
@@ -61,15 +62,17 @@ export async function loadAction({
   log,
   kbnClient,
   performance,
+  dataOnly,
 }: {
   inputDir: string;
   skipExisting: boolean;
   useCreate: boolean;
   docsOnly?: boolean;
   client: Client;
   log: ToolingLog;
-  kbnClient: KbnClient;
+  kbnClient?: KbnClient;
   performance?: LoadActionPerfOptions;
+  dataOnly?: boolean;
 }) {
   const name = relative(REPO_ROOT, inputDir);
   const isArchiveInExceptionList = soOverrideAllowedList.includes(name);
@@ -78,7 +81,7 @@ export async function loadAction({
   }
   const stats = createStats(name, log);
   const files = prioritizeMappings(await readDirectory(inputDir));
-  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
+  const kibanaPluginIds = dataOnly ? [] : await kbnClient!.plugins.getEnabledIds();
   const targetsWithoutIdGeneration: string[] = [];
 
   // a single stream that emits records from all archive files, in
@@ -123,6 +126,17 @@ export async function loadAction({
   progress.deactivate();
   const result = stats.toJSON();
 
+  if (dataOnly) {
+    const soIndices = Object.keys(result).filter(isSavedObjectIndex);
+    if (soIndices.length > 0) {
+      throw new Error(
+        `esArchiver is in dataOnly mode and cannot load saved object indices (${soIndices.join(
+          ', '
+        )}). ` + 'Use kbnArchiver to manage Kibana saved objects instead.'
+      );
+    }
+  }
+
   const indicesWithDocs: string[] = [];
   for (const [index, { docs }] of Object.entries(result)) {
     if (docs && docs.indexed > 0) {
@@ -142,8 +156,8 @@ export async function loadAction({
   );
 
   // If we affected saved objects indices, we need to ensure they are migrated...
-  if (Object.keys(result).some((k) => k.startsWith(MAIN_SAVED_OBJECT_INDEX))) {
-    await migrateSavedObjectIndices(kbnClient);
+  if (!dataOnly && Object.keys(result).some((k) => k.startsWith(MAIN_SAVED_OBJECT_INDEX))) {
+    await migrateSavedObjectIndices(kbnClient!);
     log.debug('[%s] Migrated Kibana index after loading Kibana data', name);
 
     if (kibanaPluginIds.includes('spaces')) {

src/platform/packages/shared/kbn-es-archiver/src/actions/unload.ts

@@ -30,12 +30,11 @@ export async function unloadAction({
   inputDir,
   client,
   log,
-  kbnClient,
 }: {
   inputDir: string;
   client: Client;
   log: ToolingLog;
-  kbnClient: KbnClient;
+  kbnClient?: KbnClient;
 }) {
   const name = relative(REPO_ROOT, inputDir);
   const stats = createStats(name, log);

src/platform/packages/shared/kbn-es-archiver/src/es_archiver.test.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+jest.mock('./actions', () => ({
+  loadAction: jest.fn().mockResolvedValue({}),
+  saveAction: jest.fn(),
+  unloadAction: jest.fn(),
+  rebuildAllAction: jest.fn(),
+  emptyKibanaIndexAction: jest.fn(),
+  editAction: jest.fn(),
+}));
+
+import type { Client } from '@elastic/elasticsearch';
+import type { KbnClient } from '@kbn/test';
+import { ToolingLog } from '@kbn/tooling-log';
+import { EsArchiver } from './es_archiver';
+import { loadAction } from './actions';
+
+const createMockClient = () => ({} as unknown as Client);
+const createMockKbnClient = () => ({} as unknown as KbnClient);
+const log = new ToolingLog();
+
+describe('EsArchiver', () => {
+  describe('constructor', () => {
+    it('should require kbnClient when dataOnly is not set', () => {
+      expect(
+        () =>
+          new EsArchiver({
+            client: createMockClient(),
+            log,
+          })
+      ).toThrow('kbnClient is required when dataOnly is not enabled');
+    });
+
+    it('should require kbnClient when dataOnly is false', () => {
+      expect(
+        () =>
+          new EsArchiver({
+            client: createMockClient(),
+            log,
+            dataOnly: false,
+          })
+      ).toThrow('kbnClient is required when dataOnly is not enabled');
+    });
+
+    it('should not require kbnClient when dataOnly is true', () => {
+      expect(
+        () =>
+          new EsArchiver({
+            client: createMockClient(),
+            log,
+            dataOnly: true,
+          })
+      ).not.toThrow();
+    });
+
+    it('should accept kbnClient with dataOnly false', () => {
+      expect(
+        () =>
+          new EsArchiver({
+            client: createMockClient(),
+            log,
+            kbnClient: createMockKbnClient(),
+            dataOnly: false,
+          })
+      ).not.toThrow();
+    });
+
+    it('should accept kbnClient without dataOnly (backward compat)', () => {
+      expect(
+        () =>
+          new EsArchiver({
+            client: createMockClient(),
+            log,
+            kbnClient: createMockKbnClient(),
+          })
+      ).not.toThrow();
+    });
+  });
+
+  describe('load', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should pass dataOnly=true to loadAction when in dataOnly mode', async () => {
+      const client = createMockClient();
+      const archiver = new EsArchiver({
+        client,
+        log,
+        dataOnly: true,
+        baseDir: __dirname,
+      });
+
+      (loadAction as jest.Mock).mockResolvedValue({});
+      await archiver.load('lib');
+
+      expect(loadAction as jest.Mock).toHaveBeenCalledTimes(1);
+      expect((loadAction as jest.Mock).mock.calls[0][0]).toMatchObject({
+        dataOnly: true,
+        kbnClient: undefined,
+      });
+    });
+
+    it('should pass dataOnly=false to loadAction when not in dataOnly mode', async () => {
+      const client = createMockClient();
+      const kbnClient = createMockKbnClient();
+      const archiver = new EsArchiver({
+        client,
+        log,
+        kbnClient,
+        baseDir: __dirname,
+      });
+
+      (loadAction as jest.Mock).mockResolvedValue({});
+      await archiver.load('lib');
+
+      expect(loadAction as jest.Mock).toHaveBeenCalledTimes(1);
+      expect((loadAction as jest.Mock).mock.calls[0][0]).toMatchObject({
+        dataOnly: false,
+        kbnClient,
+      });
+    });
+  });
+
+  describe('emptyKibanaIndex', () => {
+    it('should throw in dataOnly mode', async () => {
+      const archiver = new EsArchiver({
+        client: createMockClient(),
+        log,
+        dataOnly: true,
+      });
+
+      await expect(archiver.emptyKibanaIndex()).rejects.toThrow(
+        'emptyKibanaIndex is not supported in dataOnly mode'
+      );
+    });
+  });
+});

src/platform/packages/shared/kbn-es-archiver/src/es_archiver.ts

@@ -28,16 +28,27 @@ interface Options {
   client: Client;
   baseDir?: string;
   log: ToolingLog;
-  kbnClient: KbnClient;
+  kbnClient?: KbnClient;
+  /**
+   * When true, `kbnClient` is not required and loading archives that contain
+   * saved-object indices (.kibana*) will throw an error. Intended for Scout tests and
+   * linked CPS projects that should only ingest pure ES data.
+   */
+  dataOnly?: boolean;
 }
 
 export class EsArchiver {
   private readonly client: Client;
   private readonly baseDir: string;
   private readonly log: ToolingLog;
-  private readonly kbnClient: KbnClient;
+  private readonly kbnClient: KbnClient | undefined;
+  private readonly dataOnly: boolean;
 
   constructor(options: Options) {
+    this.dataOnly = options.dataOnly ?? false;
+    if (!this.dataOnly && !options.kbnClient) {
+      throw new Error('kbnClient is required when dataOnly is not enabled');
+    }
     this.client = options.client;
     this.baseDir = options.baseDir ?? REPO_ROOT;
     this.log = options.log;
@@ -108,6 +119,7 @@ export class EsArchiver {
         log: this.log,
         kbnClient: this.kbnClient,
         performance,
+        dataOnly: this.dataOnly,
       })
     );
   }
@@ -123,7 +135,6 @@ export class EsArchiver {
         inputDir: this.findArchive(path),
         client: this.client,
         log: this.log,
-        kbnClient: this.kbnClient,
       })
     );
   }
@@ -169,6 +180,13 @@ export class EsArchiver {
    * Cleanup saved object indices, preserving the space:default saved object.
    */
   async emptyKibanaIndex() {
+    if (this.dataOnly) {
+      throw new Error(
+        'emptyKibanaIndex is not supported in dataOnly mode. ' +
+          'Use kbnArchiver to manage Kibana saved objects instead.'
+      );
+    }
+
     return await withSpan('es_archiver empty_kibana_index', () =>
       emptyKibanaIndexAction({
         client: this.client,

src/platform/packages/shared/kbn-es-archiver/src/lib/index.ts

@@ -21,6 +21,7 @@ export {
   migrateSavedObjectIndices,
   cleanSavedObjectIndices,
   createDefaultSpace,
+  isSavedObjectIndex,
 } from './indices';
 
 export { createFilterRecordsStream } from './records';

src/platform/packages/shared/kbn-es-archiver/src/lib/indices/index.ts

@@ -15,4 +15,5 @@ export {
   deleteSavedObjectIndices,
   cleanSavedObjectIndices,
   createDefaultSpace,
+  isSavedObjectIndex,
 } from './kibana_index';

src/platform/packages/shared/kbn-scout/src/common/services/es_archiver.ts

@@ -10,17 +10,17 @@
 import { EsArchiver } from '@kbn/es-archiver';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { ScoutLogger } from './logger';
-import type { EsClient, KbnClient } from '../../types';
+import type { EsClient } from '../../types';
 
 let esArchiverInstance: EsArchiver | undefined;
 
-export function getEsArchiver(esClient: EsClient, kbnClient: KbnClient, log: ScoutLogger) {
+export function getEsArchiver(esClient: EsClient, log: ScoutLogger) {
   if (!esArchiverInstance) {
     esArchiverInstance = new EsArchiver({
       log,
       client: esClient,
-      kbnClient,
       baseDir: REPO_ROOT,
+      dataOnly: true,
     });
 
     log.serviceLoaded('esArchiver');

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/scope/worker/es_archiver.ts

@@ -36,8 +36,8 @@ export const esArchiverFixture = coreWorkerFixtures.extend<{}, { esArchiver: EsA
    * we only expose capability to ingest the data indexes.
    */
   esArchiver: [
-    ({ log, esClient, kbnClient }, use) => {
-      const esArchiverInstance = getEsArchiver(esClient, kbnClient, log);
+    ({ log, esClient }, use) => {
+      const esArchiverInstance = getEsArchiver(esClient, log);
       const loadIfNeeded = async (name: string, performance?: LoadActionPerfOptions | undefined) =>
         esArchiverInstance!.loadIfNeeded(name, performance);
 

src/platform/packages/shared/kbn-scout/src/playwright/global_hooks/data_ingestion.ts

@@ -13,7 +13,6 @@ import {
   createScoutConfig,
   measurePerformanceAsync,
   getEsClient,
-  getKbnClient,
 } from '../../common';
 import { ScoutLogger } from '../../common/services/logger';
 import type { ScoutTestOptions } from '../types';
@@ -33,8 +32,7 @@ export async function ingestTestDataHook(config: FullConfig, archives: string[])
     const serversConfigDir = projectUse.serversConfigDir;
     const scoutConfig = createScoutConfig(serversConfigDir, configName, log);
     const esClient = getEsClient(scoutConfig, log);
-    const kbnClient = getKbnClient(scoutConfig, log);
-    const esArchiver = getEsArchiver(esClient, kbnClient, log);
+    const esArchiver = getEsArchiver(esClient, log);
 
     log.debug('[setup] loading test data (only if indexes do not exist)...');
     for (const archive of archives) {