[Discover Sessions as Code] Discover search embeddable: API schema, transforms, and feature flag

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -168,7 +168,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "risk-engine-configuration": "9d54f733fb2bd08978c7059d71e77741574dc2616823745501742d34816a408c",
         "rules-settings": "8a2c9bb693534c347d2a8f99fa50e6e8d3ef0abb567a53284a7545d58f250903",
         "sample-data-telemetry": "4c102e89bdcaee1ccc887d1709c7e176c05f25b4c5ac14c3d013b58fbfd806ac",
-        "search": "b2ce1e4bfa6b0bc7f6170bdf49013ad05b99e4e57d110d6371a80861efe2ce66",
+        "search": "1df1f6389931c8f004f26be006c70bec95a06a790f3b25fbc601b47dfb03f50b",
         "search-session": "7648bc4e0f7030ea596ab20690f2b6256ce071a206dea4912a00363737f10ba6",
         "search-telemetry": "c152fc7e66d5ac7907e81c0926be9c219a15181e10b418b2fbb86bab2760627c",
         "search_playground": "4b8d7d2bc55cafc3ff7ad479d8d1283568f3edab502d048c49513c3be33e41bc",
@@ -1116,14 +1116,14 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "search|global: ce649a79d99c5ff5eb68d544635428ef87946d84",
         "search|mappings: 432d4dfdb5a33ce29d00ccdcfcda70d7c5f94b52",
         "search|schemas: 8d6477e08dfdf20335752a69994646f9da90741f",
-        "search|10.12.0: 7442d4976ccce79029a1019106a6795d0f21e3232e4c9d1e51eed7bf165f1298",
-        "search|10.11.0: 5bc01080ab7e1843af8012fefa4b6360e0b084f3958db79fc4635bda052ce04b",
-        "search|10.10.0: 45de2f1aac057a1115e9e6cfbeaf3ba392939f5f5df85519712eae07e93a20e2",
-        "search|10.9.0: 4235a3d4c888ecfcb8f6e006c8f07f737ef3123496a3215497b7b3112eca0112",
-        "search|10.8.0: d3a125618c5f17de1514e3b69d9575e563a99746aea9e0085f568e325f20305a",
-        "search|10.7.0: 55810e03d060c87c13f34d0f14e6e0ef03fdbff89ad30c504496f1a196fd2d25",
-        "search|10.6.0: a76d1a1200c9279cf21e5752663c4e22eb683473d3eafd5e6ef656f1cbd90e8d",
-        "search|10.5.0: b3f27b015d8987732f53e695bbc8a7c337448aee7f927a0621e7cab09e15abcd",
+        "search|10.12.0: 3ce6b6ff77eab9595c76c6430b6bea4e8f29dc4b5a2635da5a9fd8e814b200ea",
+        "search|10.11.0: 89719bc8fabbea6ae39c7e50b03ea2584086e116082b7f79f9cc6303c53f1f36",
+        "search|10.10.0: ba62a15ec99c9f8665965cf621da158300f400e7c2a4d6eb6b4077b4c4eec5e1",
+        "search|10.9.0: 0b932544700ea6a1186b263c4593adf6c6f7c257de702efd6fcb24ba0df67c4c",
+        "search|10.8.0: 266a781eb6b3a3f0b2ee06304db0ac7828991b370ed1f13b181f7719ea9b7afc",
+        "search|10.7.0: 2b5c1d095d1156b2c187aecd79215110f8ceb66edf5a9ce6c3e6db2bc56bf3a6",
+        "search|10.6.0: 7a3f96df2caa6871c0ad18b0c093ce761f47fe7fe147f72ecf4cddeca9442df0",
+        "search|10.5.0: bae8b969e4a8f8f4511b40cfc24a59f3eef505a8f272b04979f26a4db934c3c4",
         "search|10.4.0: 770d4bf717afcb7034b36222904d5d8d8c6358a820f3aebdf1691a24bf9f844d",
         "search|10.3.0: 9bc63a0504674773c55800b17782ad3de232f423f1ba94cdeb1c7c825c562ab8",
         "search|10.2.0: 4a62769d2137b7e415fec5262af8385989c1fed12ce9dd17249fbb661617d879",

src/platform/packages/shared/as-code/data-views-schema/index.ts

@@ -7,5 +7,16 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+export { AS_CODE_DATA_VIEW_REFERENCE_TYPE, AS_CODE_DATA_VIEW_SPEC_TYPE } from './src/constants';
+export {
+  dataViewReferenceSchema,
+  dataViewSchema,
+  dataViewSpecSchema,
+} from './src/schema_data_view';
 export { runtimeFieldSchema } from './src/schema_runtime_field';
-export type { AsCodeRuntimeField } from './src/types';
+export type {
+  AsCodeDataView,
+  AsCodeDataViewReference,
+  AsCodeDataViewSpec,
+  AsCodeRuntimeField,
+} from './src/types';

src/platform/packages/shared/as-code/data-views-schema/src/constants.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/** `type` discriminator for as-code classic-tab `data_source`: saved Kibana data view id. */
+export const AS_CODE_DATA_VIEW_REFERENCE_TYPE = 'data_view_reference' as const;
+
+/** `type` discriminator for as-code classic-tab `data_source`: inline DataViewSpec-shaped fields. */
+export const AS_CODE_DATA_VIEW_SPEC_TYPE = 'data_view_spec' as const;

src/platform/packages/shared/as-code/data-views-schema/src/schema_data_view.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { schema } from '@kbn/config-schema';
+import { AS_CODE_DATA_VIEW_REFERENCE_TYPE, AS_CODE_DATA_VIEW_SPEC_TYPE } from './constants';
+import { runtimeFieldSchema } from './schema_runtime_field';
+
+export const dataViewReferenceSchema = schema.object(
+  {
+    type: schema.literal(AS_CODE_DATA_VIEW_REFERENCE_TYPE),
+    id: schema.string({
+      meta: {
+        description:
+          'The id of the Kibana data view to use as the data source. Example: "my-data-view".',
+      },
+    }),
+  },
+  { meta: { id: 'dataViewReferenceDataSourceTypeSchema' } }
+);
+
+export const dataViewSpecSchema = schema.object(
+  {
+    type: schema.literal(AS_CODE_DATA_VIEW_SPEC_TYPE),
+    index_pattern: schema.string({
+      meta: {
+        description:
+          'The index pattern (Elasticsearch index expression) to use as the data source. Example: "my-index-*".',
+      },
+    }),
+    time_field: schema.maybe(
+      schema.string({
+        meta: {
+          description:
+            'The name of the time field in the index. Used for time-based filtering. Example: "@timestamp".',
+        },
+      })
+    ),
+    runtime_fields: schema.maybe(schema.arrayOf(runtimeFieldSchema, { maxSize: 100 })),
+  },
+  { meta: { id: 'dataViewSpecDataSourceTypeSchema' } }
+);
+
+export const dataViewSchema = schema.discriminatedUnion('type', [
+  dataViewReferenceSchema,
+  dataViewSpecSchema,
+]);

src/platform/packages/shared/as-code/data-views-schema/src/types.ts

@@ -7,6 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 import type { TypeOf } from '@kbn/config-schema';
+import type {
+  dataViewReferenceSchema,
+  dataViewSchema,
+  dataViewSpecSchema,
+} from './schema_data_view';
 import type { runtimeFieldSchema } from './schema_runtime_field';
 
 export type AsCodeRuntimeField = TypeOf<typeof runtimeFieldSchema>;
+export type AsCodeDataViewReference = TypeOf<typeof dataViewReferenceSchema>;
+export type AsCodeDataViewSpec = TypeOf<typeof dataViewSpecSchema>;
+export type AsCodeDataView = TypeOf<typeof dataViewSchema>;

src/platform/packages/shared/as-code/data-views-transforms/index.ts

@@ -7,5 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+export * from './src/from_stored_data_view';
 export * from './src/from_stored_runtime_fields';
+export * from './src/to_stored_data_view';
 export * from './src/to_stored_runtime_fields';

src/platform/packages/shared/as-code/data-views-transforms/src/from_stored_data_view.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
+import {
+  AS_CODE_DATA_VIEW_REFERENCE_TYPE,
+  AS_CODE_DATA_VIEW_SPEC_TYPE,
+  type AsCodeDataView,
+} from '@kbn/as-code-data-views-schema';
+import { fromStoredRuntimeFields } from './from_stored_runtime_fields';
+
+/**
+ * Convert a stored search-source `index` value (saved object / serialized search source)
+ * to the as-code data view shape.
+ *
+ * @param index String id (referenced data view), inline {@link DataViewSpec}, or null/undefined
+ * @returns As-code `data_source` object for classic (KQL/Lucene) tabs
+ */
+export function fromStoredDataView(
+  index: string | DataViewSpec | null | undefined
+): AsCodeDataView {
+  if (!index) throw new Error('Cannot derive data view from empty index');
+  if (typeof index === 'string') {
+    return { type: AS_CODE_DATA_VIEW_REFERENCE_TYPE, id: index };
+  }
+  if (!index.title) throw new Error('Cannot derive data view without `title` or `id`');
+  return {
+    type: AS_CODE_DATA_VIEW_SPEC_TYPE,
+    index_pattern: index.title,
+    time_field: index.timeFieldName,
+    runtime_fields: fromStoredRuntimeFields(
+      index.runtimeFieldMap,
+      index.fieldFormats,
+      index.fieldAttrs
+    ),
+  };
+}

src/platform/packages/shared/as-code/data-views-transforms/src/stored_data_source.test.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import {
+  AS_CODE_DATA_VIEW_REFERENCE_TYPE,
+  AS_CODE_DATA_VIEW_SPEC_TYPE,
+  type AsCodeDataViewReference,
+  type AsCodeDataViewSpec,
+} from '@kbn/as-code-data-views-schema';
+import { toStoredDataView } from './to_stored_data_view';
+
+describe('toStoredDataView', () => {
+  it('converts data_view_reference data_source to string id', () => {
+    const dataView: AsCodeDataViewReference = {
+      type: AS_CODE_DATA_VIEW_REFERENCE_TYPE,
+      id: 'my-data-view-id',
+    };
+    const result = toStoredDataView(dataView);
+    expect(result).toBe('my-data-view-id');
+  });
+
+  it('converts index-pattern data_source to serialized index spec', () => {
+    const dataView: AsCodeDataViewSpec = {
+      type: AS_CODE_DATA_VIEW_SPEC_TYPE,
+      index_pattern: 'my-index-*',
+      time_field: '@timestamp',
+      runtime_fields: [
+        {
+          name: 'rt',
+          type: 'keyword',
+          script: 'emit(doc["id"].value)',
+          format: { type: 'string' },
+        },
+      ],
+    };
+    const result = toStoredDataView(dataView);
+    expect(result).toEqual({
+      title: 'my-index-*',
+      timeFieldName: '@timestamp',
+      fieldFormats: {
+        rt: { id: 'string', params: undefined },
+      },
+      fieldAttrs: {
+        rt: {},
+      },
+      runtimeFieldMap: {
+        rt: {
+          type: 'keyword',
+          script: { source: 'emit(doc["id"].value)' },
+        },
+      },
+    });
+  });
+
+  it('converts index-pattern data_source without runtime fields', () => {
+    const dataView: AsCodeDataViewSpec = {
+      type: AS_CODE_DATA_VIEW_SPEC_TYPE,
+      index_pattern: 'logs-*',
+      time_field: '@timestamp',
+    };
+    const result = toStoredDataView(dataView);
+    expect(result).toEqual({
+      title: 'logs-*',
+      timeFieldName: '@timestamp',
+    });
+  });
+});

src/platform/packages/shared/as-code/data-views-transforms/src/to_stored_data_view.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
+import {
+  AS_CODE_DATA_VIEW_REFERENCE_TYPE,
+  type AsCodeDataView,
+} from '@kbn/as-code-data-views-schema';
+import {
+  toStoredFieldAttributes,
+  toStoredFieldFormats,
+  toStoredRuntimeFields,
+} from './to_stored_runtime_fields';
+
+/**
+ * Convert an as-code data view back to a stored search-source `index` value
+ * (string id for a referenced data view, or inline {@link DataViewSpec} fields).
+ *
+ * @param dataView As-code `data_source` value from classic tab state
+ * @returns Value suitable for `SerializedSearchSourceFields.index`
+ */
+export function toStoredDataView(dataView: AsCodeDataView): string | DataViewSpec {
+  if (dataView.type === AS_CODE_DATA_VIEW_REFERENCE_TYPE) return dataView.id;
+  const runtimeFieldMap = toStoredRuntimeFields(dataView.runtime_fields);
+  const fieldFormats = toStoredFieldFormats(dataView.runtime_fields);
+  const fieldAttrs = toStoredFieldAttributes(dataView.runtime_fields);
+  return {
+    title: dataView.index_pattern,
+    timeFieldName: dataView.time_field,
+    ...(runtimeFieldMap && Object.keys(runtimeFieldMap).length > 0 && { runtimeFieldMap }),
+    ...(fieldFormats && Object.keys(fieldFormats).length > 0 && { fieldFormats }),
+    ...(fieldAttrs && Object.keys(fieldAttrs).length > 0 && { fieldAttrs }),
+  };
+}

src/platform/packages/shared/kbn-saved-search-component/src/components/saved_search.tsx

@@ -12,7 +12,10 @@ import { debounceTime, distinctUntilChanged, map } from 'rxjs';
 import { isEqual } from 'lodash';
 import { EmbeddableRenderer } from '@kbn/embeddable-plugin/public';
 import { SEARCH_EMBEDDABLE_TYPE, getDefaultSort } from '@kbn/discover-utils';
-import type { SearchEmbeddableApi } from '@kbn/discover-plugin/public';
+import {
+  type SearchEmbeddableApi,
+  type SearchEmbeddablePanelApiState,
+} from '@kbn/discover-plugin/public';
 import type { SearchEmbeddableState } from '@kbn/discover-plugin/common';
 import { css } from '@emotion/react';
 import { type SavedSearch, toSavedSearchAttributes } from '@kbn/saved-search-plugin/common';
@@ -268,7 +271,7 @@ const SavedSearchComponentTable: React.FC<
   );
 
   return (
-    <EmbeddableRenderer<SearchEmbeddableState, SearchEmbeddableApi>
+    <EmbeddableRenderer<SearchEmbeddablePanelApiState, SearchEmbeddableApi>
       maybeId={undefined}
       type={SEARCH_EMBEDDABLE_TYPE}
       getParentApi={() => parentApi}

src/platform/plugins/shared/discover/common/constants.ts

@@ -48,3 +48,10 @@ export const TAB_STATE_URL_KEY = '_tab'; // `_t` is already used by Kibana for t
  */
 export const TRACES_PRODUCT_FEATURE_ID = 'discover:traces';
 export const METRICS_EXPERIENCE_PRODUCT_FEATURE_ID = 'discover:metrics-experience';
+
+/**
+ * When enabled, Discover search embeddable uses transformIn/transformOut to convert between
+ * API format (DiscoverSessionEmbeddableState) and stored format (StoredSearchEmbeddableState).
+ * When disabled, panel state is stored and loaded as-is (pre-transform behavior).
+ */
+export const EMBEDDABLE_TRANSFORMS_FEATURE_FLAG_KEY = 'discover.embeddableTransforms';

src/platform/plugins/shared/discover/common/embeddable/constants.ts

@@ -9,6 +9,19 @@
 
 import type { SavedSearchAttributes } from '@kbn/saved-search-plugin/common';
 
+/** Reference name used for the saved search saved object when the embeddable is by-reference */
+export const SAVED_SEARCH_SAVED_OBJECT_REF_NAME = 'savedObjectRef';
+
+/**
+ * Used for search embeddable transforms. The as-code API shape does not support tab id/label. When
+ * transforming from the as-code API shape back to the stored shape, we use these synthetic values
+ * to satisfy the stored shape/types.
+ */
+export const DISCOVER_SESSION_EMBEDDABLE_SYNTHETIC_TAB_ID =
+  'discover_session_embeddable_synthetic_tab_id';
+export const DISCOVER_SESSION_EMBEDDABLE_SYNTHETIC_TAB_LABEL =
+  'discover_session_embeddable_synthetic_tab_label';
+
 /** This constant refers to the parts of the saved search state that can be edited from a dashboard */
 export const EDITABLE_SAVED_SEARCH_KEYS = [
   'sort',