From e425a9542f20bd6467293cc7397d3c9d173f851f Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 11:04:33 +0200 Subject: [PATCH 01/12] track installed content packs --- .../src/helpers/index.ts | 137 +------------- .../src/helpers/index_pattern.ts | 177 ++++++++++++++++++ .../src/models/api.ts | 34 ++++ .../src/models/index.ts | 46 ++--- .../src/models/saved_object.ts | 23 +++ .../server/lib/content/content_client.ts | 82 ++++++++ .../server/lib/content/content_service.ts | 38 ++++ .../streams/server/lib/content/fields.ts | 9 + .../streams/server/lib/content/index.ts | 1 + .../server/lib/content/saved_object.ts | 62 ++++-- .../plugins/shared/streams/server/plugin.ts | 14 +- .../streams/server/routes/content/route.ts | 34 +++- .../shared/streams/server/routes/types.ts | 2 + 13 files changed, 476 insertions(+), 183 deletions(-) create mode 100644 x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts create mode 100644 x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/api.ts create mode 100644 x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts create mode 100644 x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts create mode 100644 x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts create mode 100644 x-pack/platform/plugins/shared/streams/server/lib/content/fields.ts diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index.ts index 4bd5f716a93ef..4f8c87d5340c8 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index.ts @@ -5,139 +5,4 @@ * 2.0. */ -import type { - DashboardAttributes, - SavedDashboardPanel, -} from '@kbn/dashboard-plugin/common/content_management/v2'; -import { cloneDeep, mapValues, uniq } from 'lodash'; -import { AggregateQuery, Query } from '@kbn/es-query'; -import { getIndexPatternFromESQLQuery, replaceESQLQueryIndexPattern } from '@kbn/esql-utils'; -import type { LensAttributes } from '@kbn/lens-embeddable-utils'; -import type { IndexPatternRef } from '@kbn/lens-plugin/public/types'; -import type { ContentPackSavedObject } from '../models'; - -export const INDEX_PLACEHOLDER = ''; - -export const isIndexPlaceholder = (index: string) => index.startsWith(INDEX_PLACEHOLDER); - -interface TraverseOptions { - esqlQuery(query: string): string; - indexPattern(pattern: T): T; -} - -export function findIndexPatterns(savedObject: ContentPackSavedObject) { - const patterns: string[] = []; - - locateIndexPatterns(savedObject, { - esqlQuery(query: string) { - patterns.push(...getIndexPatternFromESQLQuery(query).split(',')); - return query; - }, - indexPattern(pattern: T) { - if (pattern.title) { - patterns.push(...pattern.title.split(',')); - } - return pattern; - }, - }); - - return uniq(patterns); -} - -export function replaceIndexPatterns( - savedObject: ContentPackSavedObject, - replacements: Record -) { - return locateIndexPatterns(cloneDeep(savedObject), { - esqlQuery(query: string) { - return replaceESQLQueryIndexPattern(query, replacements); - }, - indexPattern(pattern: T) { - const updatedPattern = pattern.title - ?.split(',') - .map((index) => replacements[index] ?? index) - .join(','); - - return { - ...pattern, - name: updatedPattern, - title: updatedPattern, - }; - }, - }); -} - -function locateIndexPatterns( - object: ContentPackSavedObject, - options: TraverseOptions -): ContentPackSavedObject { - const content = object; - - if (content.type === 'index-pattern') { - content.attributes = options.indexPattern(content.attributes); - } - - if (content.type === 'dashboard') { - const attributes = content.attributes as DashboardAttributes; - const panels = (JSON.parse(attributes.panelsJSON) as SavedDashboardPanel[]).map((panel) => - traversePanel(panel, options) - ); - - attributes.panelsJSON = JSON.stringify(panels); - } - - return object; -} - -function traversePanel(panel: SavedDashboardPanel, options: TraverseOptions) { - if (panel.type === 'lens') { - const config = panel.embeddableConfig as { - query?: Query | AggregateQuery; - attributes?: LensAttributes; - }; - if (config.query && 'esql' in config.query) { - config.query.esql = options.esqlQuery(config.query.esql); - } - - if (config.attributes) { - traverseLensPanel(config.attributes as LensAttributes, options); - } - } - - return panel; -} - -function traverseLensPanel(panel: LensAttributes, options: TraverseOptions) { - const state = panel.state; - - if (state.adHocDataViews) { - state.adHocDataViews = mapValues(state.adHocDataViews, (dataView) => - options.indexPattern(dataView) - ); - } - - const { - query: stateQuery, - datasourceStates: { textBased }, - } = state; - - if (stateQuery && 'esql' in stateQuery) { - stateQuery.esql = options.esqlQuery(stateQuery.esql); - } - - if (textBased) { - Object.values(textBased.layers).forEach((layer) => { - if (layer.query?.esql) { - layer.query.esql = options.esqlQuery(layer.query.esql); - } - }); - - if ('indexPatternRefs' in textBased) { - textBased.indexPatternRefs = (textBased.indexPatternRefs as IndexPatternRef[]).map((ref) => - options.indexPattern(ref) - ); - } - } - - return panel; -} +export * from './index_pattern'; diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts new file mode 100644 index 0000000000000..57fd448f947a0 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + DashboardAttributes, + SavedDashboardPanel, +} from '@kbn/dashboard-plugin/common/content_management/v2'; +import { cloneDeep, mapValues } from 'lodash'; +import { AggregateQuery, Query } from '@kbn/es-query'; +import { getIndexPatternFromESQLQuery, replaceESQLQueryIndexPattern } from '@kbn/esql-utils'; +import type { LensAttributes } from '@kbn/lens-embeddable-utils'; +import type { IndexPatternRef } from '@kbn/lens-plugin/public/types'; +import { FieldBasedIndexPatternColumn, GenericIndexPatternColumn } from '@kbn/lens-plugin/public'; +import { TextBasedLayerColumn } from '@kbn/lens-plugin/public/datasources/form_based/esql_layer/types'; +import type { ContentPackSavedObject } from '../models'; + +export const INDEX_PLACEHOLDER = ''; + +export const isIndexPlaceholder = (index: string) => index.startsWith(INDEX_PLACEHOLDER); + +interface TraverseOptions { + esqlQuery(query: string): string; + indexPattern(pattern: T): T; + field(field: T): T; +} + +export function findConfiguration(savedObject: ContentPackSavedObject) { + const patterns: Set = new Set(); + const fields: Record = {}; + + locateConfiguration(savedObject, { + esqlQuery(query: string) { + getIndexPatternFromESQLQuery(query) + .split(',') + .forEach((p) => patterns.add(p)); + return query; + }, + indexPattern(pattern: T) { + if (pattern.title) { + pattern.title.split(',').forEach((p) => patterns.add(p)); + } + return pattern; + }, + field(field: T): T { + if ('fieldName' in field) { + const { fieldName, meta } = field as TextBasedLayerColumn; + if (meta?.esType) { + fields[fieldName] = { type: meta.esType }; + } + } else if ('sourceField' in field) { + const { sourceField, dataType } = field as FieldBasedIndexPatternColumn; + if (sourceField !== '___records___') { + fields[sourceField] = { type: dataType }; + } + } + + return field; + }, + }); + + return { patterns: [...patterns], fields }; +} + +export function replaceIndexPatterns( + savedObject: ContentPackSavedObject, + patternReplacements: Record +) { + return locateConfiguration(cloneDeep(savedObject), { + esqlQuery(query: string) { + return replaceESQLQueryIndexPattern(query, patternReplacements); + }, + indexPattern(pattern: T) { + const updatedPattern = pattern.title + ?.split(',') + .map((index) => patternReplacements[index] ?? index) + .join(','); + + return { + ...pattern, + name: updatedPattern, + title: updatedPattern, + }; + }, + field(field: any) { + return field; + }, + }); +} + +function locateConfiguration( + object: ContentPackSavedObject, + options: TraverseOptions +): ContentPackSavedObject { + const content = object; + + if (content.type === 'index-pattern') { + content.attributes = options.indexPattern(content.attributes); + } + + if (content.type === 'dashboard') { + const attributes = content.attributes as DashboardAttributes; + const panels = (JSON.parse(attributes.panelsJSON) as SavedDashboardPanel[]).map((panel) => + traversePanel(panel, options) + ); + + attributes.panelsJSON = JSON.stringify(panels); + } + + return object; +} + +function traversePanel(panel: SavedDashboardPanel, options: TraverseOptions) { + if (panel.type === 'lens') { + const config = panel.embeddableConfig as { + query?: Query | AggregateQuery; + attributes?: LensAttributes; + }; + if (config.query && 'esql' in config.query) { + config.query.esql = options.esqlQuery(config.query.esql); + } + + if (config.attributes) { + traverseLensPanel(config.attributes as LensAttributes, options); + } + } + + return panel; +} + +function traverseLensPanel(panel: LensAttributes, options: TraverseOptions) { + const state = panel.state; + + if (state.adHocDataViews) { + state.adHocDataViews = mapValues(state.adHocDataViews, (dataView) => + options.indexPattern(dataView) + ); + } + + const { + query: stateQuery, + datasourceStates: { formBased, textBased }, + } = state; + + if (stateQuery && 'esql' in stateQuery) { + stateQuery.esql = options.esqlQuery(stateQuery.esql); + } + + if (formBased) { + Object.values(formBased.layers).forEach((layer) => { + Object.entries(layer.columns).forEach(([columnId, column]) => { + layer.columns[columnId] = options.field(column); + }); + }); + } + + if (textBased) { + Object.values(textBased.layers).forEach((layer) => { + if (layer.query?.esql) { + layer.query.esql = options.esqlQuery(layer.query.esql); + } + + layer.columns = layer.columns.map((column) => options.field(column)); + }); + + if ('indexPatternRefs' in textBased) { + textBased.indexPatternRefs = (textBased.indexPatternRefs as IndexPatternRef[]).map((ref) => + options.indexPattern(ref) + ); + } + } + + return panel; +} diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/api.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/api.ts new file mode 100644 index 0000000000000..fa9fa498c7a15 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/api.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; + +export interface ContentPackIncludeObjects { + objects: { + dashboards: string[]; + }; +} + +export interface ContentPackIncludeAll { + all: {}; +} + +export type ContentPackIncludedObjects = ContentPackIncludeObjects | ContentPackIncludeAll; + +const contentPackIncludeObjectsSchema = z.object({ + objects: z.object({ dashboards: z.array(z.string()) }), +}); +const contentPackIncludeAllSchema = z.object({ all: z.strictObject({}) }); + +export const isIncludeAll = (value: ContentPackIncludedObjects): value is ContentPackIncludeAll => { + return contentPackIncludeAllSchema.safeParse(value).success; +}; + +export const contentPackIncludedObjectsSchema: z.Schema = z.union([ + contentPackIncludeObjectsSchema, + contentPackIncludeAllSchema, +]); diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/index.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/index.ts index 59f4b378d7f11..c9a64b6b1bda2 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/index.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/index.ts @@ -6,9 +6,10 @@ */ import { z } from '@kbn/zod'; -import type { SavedObject } from '@kbn/core/server'; -import type { DashboardAttributes } from '@kbn/dashboard-plugin/common/content_management/v2'; -import type { DataViewSavedObjectAttrs } from '@kbn/data-views-plugin/common/data_views'; +import { ContentPackSavedObject } from './saved_object'; + +export * from './api'; +export * from './saved_object'; export interface ContentPackManifest { name: string; @@ -22,38 +23,23 @@ export const contentPackManifestSchema: z.Schema = z.object version: z.string(), }); -export interface ContentPack extends ContentPackManifest { - entries: ContentPackEntry[]; +export function isContentPackSavedObject(entry: ContentPackEntry): entry is ContentPackSavedObject { + return ['dashboard', 'index-pattern'].includes(entry.type); } -type ContentPackDashboard = SavedObject; -type ContentPackDataView = SavedObject; -export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView; - export type ContentPackEntry = ContentPackSavedObject; -export interface ContentPackIncludeObjects { - objects: { - dashboards: string[]; - }; +export interface ContentPack extends ContentPackManifest { + entries: ContentPackEntry[]; } -export interface ContentPackIncludeAll { - all: {}; +export interface ContentPackPreviewEntry { + type: string; + id: string; + title: string; + errors: Array<{ severity: 'fatal' | 'warning'; message: string }>; } -export type ContentPackIncludedObjects = ContentPackIncludeObjects | ContentPackIncludeAll; - -const contentPackIncludeObjectsSchema = z.object({ - objects: z.object({ dashboards: z.array(z.string()) }), -}); -const contentPackIncludeAllSchema = z.object({ all: z.strictObject({}) }); - -export const isIncludeAll = (value: ContentPackIncludedObjects): value is ContentPackIncludeAll => { - return contentPackIncludeAllSchema.safeParse(value).success; -}; - -export const contentPackIncludedObjectsSchema: z.Schema = z.union([ - contentPackIncludeObjectsSchema, - contentPackIncludeAllSchema, -]); +export interface ContentPackPreview extends ContentPackManifest { + entries: ContentPackPreviewEntry[]; +} diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts new file mode 100644 index 0000000000000..e95b70bf8faff --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject } from '@kbn/core/server'; +import type { DashboardAttributes } from '@kbn/dashboard-plugin/common/content_management/v2'; +import type { DataViewSavedObjectAttrs } from '@kbn/data-views-plugin/common/data_views'; + +type ContentPackDashboard = SavedObject; +type ContentPackDataView = SavedObject; +export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView; + +export type SavedObjectLink = { + source_id: string; + target_id: string; +}; + +export type SavedObjectLinkWithReferences = SavedObjectLink & { references: SavedObjectLink[] }; + +export type ContentPackSavedObjectLinks = { dashboards: SavedObjectLinkWithReferences[] }; diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts new file mode 100644 index 0000000000000..088f5da1e56d6 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + ContentPackSavedObjectLinks, + SavedObjectLinkWithReferences, +} from '@kbn/content-packs-schema'; +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { IStorageClient, IndexStorageSettings, types } from '@kbn/storage-adapter'; +import objectHash from 'object-hash'; +import { CONTENT_NAME, STREAM_NAME } from './fields'; + +export const contentStorageSettings = { + name: '.kibana_streams_content_packs', + schema: { + properties: { + [STREAM_NAME]: types.keyword(), + [CONTENT_NAME]: types.keyword(), + dashboards: types.object(), + 'dashboards.source_id': types.keyword(), + 'dashboards.target_id': types.keyword(), + 'dashboards.references': types.object(), + 'dashboards.references.source_id': types.keyword(), + 'dashboards.references.target_id': types.keyword(), + }, + }, +} satisfies IndexStorageSettings; + +export type ContentStorageSettings = typeof contentStorageSettings; + +export type StoredContentPack = { + [STREAM_NAME]: string; + [CONTENT_NAME]: string; + dashboards: SavedObjectLinkWithReferences[]; +}; + +export class ContentClient { + constructor( + private readonly clients: { + storageClient: IStorageClient; + soClient: SavedObjectsClientContract; + } + ) {} + + async getStoredContentPacks(streamName: string) { + const response = await this.clients.storageClient.search({ + size: 10_000, + track_total_hits: false, + query: { + bool: { + filter: [{ term: { [STREAM_NAME]: streamName } }], + }, + }, + }); + + return response; + } + + async getStoredContentPack(streamName: string, contentName: string) { + const id = objectHash({ streamName, contentName }); + const response = await this.clients.storageClient.get({ id }); + return response._source!; + } + + async upsertStoredContentPack( + streamName: string, + content: { name: string } & ContentPackSavedObjectLinks + ) { + const id = objectHash({ streamName, contentName: content.name }); + await this.clients.storageClient.index({ + id, + document: { + [STREAM_NAME]: streamName, + [CONTENT_NAME]: content.name, + dashboards: content.dashboards, + }, + }); + } +} diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts new file mode 100644 index 0000000000000..8a88b3d6dda9c --- /dev/null +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import { StorageIndexAdapter } from '@kbn/storage-adapter'; +import { StreamsPluginStartDependencies } from '../../types'; +import { + ContentClient, + ContentStorageSettings, + StoredContentPack, + contentStorageSettings, +} from './content_client'; + +export class ContentService { + constructor( + private readonly coreSetup: CoreSetup, + private readonly logger: Logger + ) {} + + async getClientWithRequest({ request }: { request: KibanaRequest }): Promise { + const [coreStart] = await this.coreSetup.getStartServices(); + + const adapter = new StorageIndexAdapter( + coreStart.elasticsearch.client.asInternalUser, + this.logger.get('content'), + contentStorageSettings + ); + + return new ContentClient({ + storageClient: adapter.getClient(), + soClient: coreStart.savedObjects.getScopedClient(request), + }); + } +} diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/fields.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/fields.ts new file mode 100644 index 0000000000000..c6114f8467495 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/fields.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const STREAM_NAME = 'stream.name'; +export const CONTENT_NAME = 'content.name'; diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/index.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/index.ts index 362e55cfdbd71..e51e2e5302775 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/index.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/index.ts @@ -7,3 +7,4 @@ export * from './archive'; export * from './saved_object'; +export * from './fields'; diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index 7cf998614ba45..84be0415a5919 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -9,8 +9,9 @@ import { v4 } from 'uuid'; import { ContentPackIncludedObjects, ContentPackSavedObject, + ContentPackSavedObjectLinks, INDEX_PLACEHOLDER, - findIndexPatterns, + findConfiguration, isIncludeAll, replaceIndexPatterns, } from '@kbn/content-packs-schema'; @@ -27,7 +28,7 @@ export function prepareForExport({ }) { return savedObjects.map((object) => { if (object.type === 'dashboard' || object.type === 'index-pattern') { - const patterns = findIndexPatterns(object); + const { patterns } = findConfiguration(object); const replacements = { ...replacedPatterns.reduce((acc, pattern) => { acc[pattern] = INDEX_PLACEHOLDER; @@ -52,10 +53,12 @@ export function prepareForImport({ savedObjects, include, target, + links, }: { savedObjects: ContentPackSavedObject[]; include: ContentPackIncludedObjects; target: string; + links: ContentPackSavedObjectLinks; }) { const uniqObjects = uniqBy( savedObjects @@ -74,7 +77,7 @@ export function prepareForImport({ ]), ({ id }) => id ).map((object) => { - const patterns = findIndexPatterns(object); + const { patterns } = findConfiguration(object); const replacements = patterns .filter((pattern) => pattern.startsWith(INDEX_PLACEHOLDER)) .reduce((acc, pattern) => { @@ -85,24 +88,31 @@ export function prepareForImport({ return replaceIndexPatterns(object, replacements); }); - return updateIds(uniqObjects); + return updateIds(uniqObjects, links); } -export function updateIds(savedObjects: ContentPackSavedObject[]) { - const idReplacements = savedObjects.reduce((acc, object) => { - acc[object.id] = v4(); - return acc; - }, {} as Record); +export function updateIds( + savedObjects: ContentPackSavedObject[], + links: ContentPackSavedObjectLinks +) { + const existingLinks = links.dashboards.flatMap((ref) => [ref, ...ref.references]); + const targetId = (sourceId: string) => { + const link = existingLinks.find(({ source_id }) => source_id === sourceId); + if (!link) { + throw new Error(`link for [${sourceId}] was not generated`); + } + return link.target_id; + }; savedObjects.forEach((object) => { - object.id = idReplacements[object.id]; + object.id = targetId(object.id); object.references.forEach((ref) => { // only update the id if the reference is included in the content pack. // a missing reference is not necessarily an error condition since it could // point to a pre existing saved object, for example logs-* and metrics-* // data views if (savedObjects.find((so) => so.id === ref.id)) { - ref.id = idReplacements[ref.id]; + ref.id = targetId(ref.id); } }); }); @@ -110,6 +120,36 @@ export function updateIds(savedObjects: ContentPackSavedObject[]) { return savedObjects; } +// when we import a saved object into a stream we create a copy of the source +// object with a new identifier. a saved object link stores the source identifier +// of an imported object which allows overwriting already imported objects when +// (re)importing a content pack +export function savedObjectLinks( + savedObjects: ContentPackSavedObject[], + existingLinks: ContentPackSavedObjectLinks +): ContentPackSavedObjectLinks { + const dashboards = savedObjects + .filter((object) => object.type === 'dashboard') + .map((object) => { + const existingLink = existingLinks.dashboards.find( + ({ source_id }) => source_id === object.id + ); + + return { + source_id: object.id, + target_id: existingLink?.target_id ?? v4(), + references: object.references.map((ref) => ({ + source_id: ref.id, + target_id: + existingLink?.references.find((existingRef) => ref.id === existingRef.source_id) + ?.target_id ?? v4(), + })), + }; + }); + + return { dashboards }; +} + export function referenceManagedIndexPattern(savedObjects: ContentPackSavedObject[]) { return savedObjects.some((object) => object.references.some( diff --git a/x-pack/platform/plugins/shared/streams/server/plugin.ts b/x-pack/platform/plugins/shared/streams/server/plugin.ts index c1fbecfa1c5fc..0af7acac066c1 100644 --- a/x-pack/platform/plugins/shared/streams/server/plugin.ts +++ b/x-pack/platform/plugins/shared/streams/server/plugin.ts @@ -26,6 +26,7 @@ import { AssetService } from './lib/streams/assets/asset_service'; import { RouteHandlerScopedClients } from './routes/types'; import { StreamsService } from './lib/streams/service'; import { StreamsTelemetryService } from './lib/telemetry/service'; +import { ContentService } from './lib/content/content_service'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface StreamsPluginSetup {} @@ -71,6 +72,7 @@ export class StreamsPlugin const assetService = new AssetService(core, this.logger); const streamsService = new StreamsService(core, this.logger, this.isDev); + const contentService = new ContentService(core, this.logger); registerRoutes({ repository: streamsRouteRepository, @@ -83,9 +85,10 @@ export class StreamsPlugin }: { request: KibanaRequest; }): Promise => { - const [[coreStart, pluginsStart], assetClient] = await Promise.all([ + const [[coreStart, pluginsStart], assetClient, contentClient] = await Promise.all([ core.getStartServices(), assetService.getClientWithRequest({ request }), + contentService.getClientWithRequest({ request }), ]); const streamsClient = await streamsService.getClientWithRequest({ request, assetClient }); @@ -94,7 +97,14 @@ export class StreamsPlugin const soClient = coreStart.savedObjects.getScopedClient(request); const inferenceClient = pluginsStart.inference.getClient({ request }); - return { scopedClusterClient, soClient, assetClient, streamsClient, inferenceClient }; + return { + scopedClusterClient, + soClient, + assetClient, + streamsClient, + inferenceClient, + contentClient, + }; }, }, core, diff --git a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts index 89eb7149f5cb0..a1bedb1e09671 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts @@ -6,6 +6,7 @@ */ import { Readable } from 'stream'; +import { isNotFoundError } from '@kbn/es-errors'; import { z } from '@kbn/zod'; import { createConcatStream, createListStream, createPromiseFromStreams } from '@kbn/utils'; import { installManagedIndexPattern } from '@kbn/fleet-plugin/server/services/epm/kibana/assets/install'; @@ -20,12 +21,16 @@ import { createServerRoute } from '../create_server_route'; import { StatusError } from '../../lib/streams/errors/status_error'; import { ASSET_ID, ASSET_TYPE } from '../../lib/streams/assets/fields'; import { + CONTENT_NAME, + STREAM_NAME, generateArchive, parseArchive, prepareForExport, prepareForImport, referenceManagedIndexPattern, + savedObjectLinks, } from '../../lib/content'; +import { StoredContentPack } from '../../lib/content/content_client'; const MAX_CONTENT_PACK_SIZE_BYTES = 1024 * 1024 * 5; // 5MB @@ -140,18 +145,34 @@ const importContentRoute = createServerRoute({ }, }, async handler({ params, request, getScopedClients, context }) { - const { assetClient, soClient, streamsClient } = await getScopedClients({ request }); + const { assetClient, soClient, streamsClient, contentClient } = await getScopedClients({ + request, + }); + const importer = (await context.core).savedObjects.getImporter(soClient); await streamsClient.ensureStream(params.path.name); const contentPack = await parseArchive(params.body.content); + const storedContentPack = await contentClient + .getStoredContentPack(params.path.name, contentPack.name) + .catch((err) => { + if (isNotFoundError(err)) { + return { + [STREAM_NAME]: params.path.name, + [CONTENT_NAME]: contentPack.name, + dashboards: [], + } as StoredContentPack; + } + + throw err; + }); - const importer = (await context.core).savedObjects.getImporter(soClient); - + const links = savedObjectLinks(contentPack.entries, storedContentPack); const savedObjects = prepareForImport({ target: params.path.name, include: params.body.include, savedObjects: contentPack.entries, + links, }); if (referenceManagedIndexPattern(savedObjects)) { @@ -169,12 +190,17 @@ const importContentRoute = createServerRoute({ overwrite: true, }); + await contentClient.upsertStoredContentPack(params.path.name, { + name: contentPack.name, + ...links, + }); + const createdAssets: Array> = successResults ?.filter((savedObject) => savedObject.type === 'dashboard') .map((dashboard) => ({ [ASSET_TYPE]: 'dashboard', - [ASSET_ID]: dashboard.destinationId ?? dashboard.id, + [ASSET_ID]: dashboard.id, })) ?? []; if (createdAssets.length > 0) { diff --git a/x-pack/platform/plugins/shared/streams/server/routes/types.ts b/x-pack/platform/plugins/shared/streams/server/routes/types.ts index b9532ca4abd59..70aa21c7e40b7 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/types.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/types.ts @@ -15,6 +15,7 @@ import { AssetService } from '../lib/streams/assets/asset_service'; import { AssetClient } from '../lib/streams/assets/asset_client'; import { StreamsClient } from '../lib/streams/client'; import { StreamsTelemetryClient } from '../lib/telemetry/client'; +import { ContentClient } from '../lib/content/content_client'; type GetScopedClients = ({ request, @@ -28,6 +29,7 @@ export interface RouteHandlerScopedClients { assetClient: AssetClient; streamsClient: StreamsClient; inferenceClient: InferenceClient; + contentClient: ContentClient; } export interface RouteDependencies { From 5072921b60e4de0bbec49abc969865951377b35c Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 11:48:42 +0200 Subject: [PATCH 02/12] add test --- .../apis/observability/streams/content.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/content.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/content.ts index d246fca391db3..560b385468570 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/content.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/content.ts @@ -12,7 +12,7 @@ import { ContentPack, ContentPackSavedObject, INDEX_PLACEHOLDER, - findIndexPatterns, + findConfiguration, } from '@kbn/content-packs-schema'; import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; import { @@ -88,7 +88,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('puts placeholders for patterns matching the source stream', async () => { expect(contentPack.entries.length).to.eql(2); contentPack.entries.forEach((entry) => { - const patterns = findIndexPatterns(entry); + const { patterns } = findConfiguration(entry); expect(patterns).to.eql([INDEX_PLACEHOLDER]); }); }); @@ -123,6 +123,20 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { expect(stream.dashboards).to.eql([response.created[0]['asset.id']]); }); + it('does not duplicate objects when re-importing a content pack', async () => { + const archive = await generateArchive(contentPack, contentPack.entries); + const response = await importContent(apiClient, 'logs.importstream', { + include: { all: {} }, + content: Readable.from(archive), + }); + + expect(response.errors.length).to.be(0); + expect(response.created.length).to.be(1); + + const stream = await getStream(apiClient, 'logs.importstream'); + expect(stream.dashboards).to.eql([response.created[0]['asset.id']]); + }); + it('replaces placeholders with target stream pattern', async () => { const stream = await getStream(apiClient, 'logs.importstream'); const dashboard = await kibanaServer.savedObjects.get({ @@ -136,7 +150,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); [dashboard, indexPattern].forEach((object) => { - const patterns = findIndexPatterns(object as ContentPackSavedObject); + const { patterns } = findConfiguration(object as ContentPackSavedObject); expect(patterns).to.eql(['logs.importstream']); }); }); From 3b5df02ea273d30968db3a6fbe48059cca6adfe0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:50:28 +0000 Subject: [PATCH 03/12] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../kbn-content-packs-schema/src/models/saved_object.ts | 8 +++++--- .../shared/streams/server/lib/content/content_client.ts | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts index e95b70bf8faff..49a0886da0470 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts @@ -13,11 +13,13 @@ type ContentPackDashboard = SavedObject; type ContentPackDataView = SavedObject; export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView; -export type SavedObjectLink = { +export interface SavedObjectLink { source_id: string; target_id: string; -}; +} export type SavedObjectLinkWithReferences = SavedObjectLink & { references: SavedObjectLink[] }; -export type ContentPackSavedObjectLinks = { dashboards: SavedObjectLinkWithReferences[] }; +export interface ContentPackSavedObjectLinks { + dashboards: SavedObjectLinkWithReferences[]; +} diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts index 088f5da1e56d6..5ff4c1de3a23f 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts @@ -31,11 +31,11 @@ export const contentStorageSettings = { export type ContentStorageSettings = typeof contentStorageSettings; -export type StoredContentPack = { +export interface StoredContentPack { [STREAM_NAME]: string; [CONTENT_NAME]: string; dashboards: SavedObjectLinkWithReferences[]; -}; +} export class ContentClient { constructor( From f5872674bdc91ec612e252ddd7d27dd8a4420768 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 12:06:37 +0200 Subject: [PATCH 04/12] import type --- .../kbn-content-packs-schema/src/helpers/index_pattern.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts index 57fd448f947a0..2ccd5ae36cbc9 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts @@ -14,8 +14,11 @@ import { AggregateQuery, Query } from '@kbn/es-query'; import { getIndexPatternFromESQLQuery, replaceESQLQueryIndexPattern } from '@kbn/esql-utils'; import type { LensAttributes } from '@kbn/lens-embeddable-utils'; import type { IndexPatternRef } from '@kbn/lens-plugin/public/types'; -import { FieldBasedIndexPatternColumn, GenericIndexPatternColumn } from '@kbn/lens-plugin/public'; -import { TextBasedLayerColumn } from '@kbn/lens-plugin/public/datasources/form_based/esql_layer/types'; +import type { + FieldBasedIndexPatternColumn, + GenericIndexPatternColumn, +} from '@kbn/lens-plugin/public'; +import type { TextBasedLayerColumn } from '@kbn/lens-plugin/public/datasources/form_based/esql_layer/types'; import type { ContentPackSavedObject } from '../models'; export const INDEX_PLACEHOLDER = ''; From facf1dfc3917842a4cb0bfdfc72cebc0f02ed6e1 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 12:43:32 +0200 Subject: [PATCH 05/12] lint --- .../kbn-content-packs-schema/src/models/saved_object.ts | 4 ++-- .../shared/streams/server/lib/content/saved_object.ts | 6 ++---- .../plugins/shared/streams/server/routes/content/route.ts | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts index 49a0886da0470..45a1ed59e7dd3 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts @@ -9,8 +9,8 @@ import type { SavedObject } from '@kbn/core/server'; import type { DashboardAttributes } from '@kbn/dashboard-plugin/common/content_management/v2'; import type { DataViewSavedObjectAttrs } from '@kbn/data-views-plugin/common/data_views'; -type ContentPackDashboard = SavedObject; -type ContentPackDataView = SavedObject; +type ContentPackDashboard = SavedObject & { type: 'dashboard' }; +type ContentPackDataView = SavedObject & { type: 'index-pattern' }; export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView; export interface SavedObjectLink { diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index 84be0415a5919..6816123f049c1 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -97,7 +97,7 @@ export function updateIds( ) { const existingLinks = links.dashboards.flatMap((ref) => [ref, ...ref.references]); const targetId = (sourceId: string) => { - const link = existingLinks.find(({ source_id }) => source_id === sourceId); + const link = existingLinks.find(({ source_id: id }) => id === sourceId); if (!link) { throw new Error(`link for [${sourceId}] was not generated`); } @@ -131,9 +131,7 @@ export function savedObjectLinks( const dashboards = savedObjects .filter((object) => object.type === 'dashboard') .map((object) => { - const existingLink = existingLinks.dashboards.find( - ({ source_id }) => source_id === object.id - ); + const existingLink = existingLinks.dashboards.find(({ source_id: id }) => id === object.id); return { source_id: object.id, diff --git a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts index a1bedb1e09671..b6300c3b5c4f1 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts @@ -11,7 +11,7 @@ import { z } from '@kbn/zod'; import { createConcatStream, createListStream, createPromiseFromStreams } from '@kbn/utils'; import { installManagedIndexPattern } from '@kbn/fleet-plugin/server/services/epm/kibana/assets/install'; import { - ContentPackEntry, + ContentPackSavedObject, contentPackIncludedObjectsSchema, isIncludeAll, } from '@kbn/content-packs-schema'; @@ -91,7 +91,7 @@ const exportContentRoute = createServerRoute({ includeReferencesDeep: true, }); - const savedObjects: ContentPackEntry[] = await createPromiseFromStreams([ + const savedObjects: ContentPackSavedObject[] = await createPromiseFromStreams([ exportStream, createConcatStream([]), ]); From baa44c49ba8e14d4b508d74183536d6b300ab9e8 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 13:15:31 +0200 Subject: [PATCH 06/12] lint --- .../export_content_pack_flyout.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_dashboards_view/export_content_pack_flyout.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_dashboards_view/export_content_pack_flyout.tsx index b8f4bd260724a..e108e691bb61a 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_dashboards_view/export_content_pack_flyout.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_dashboards_view/export_content_pack_flyout.tsx @@ -12,7 +12,7 @@ import { IngestStreamGetResponse } from '@kbn/streams-schema'; import { ContentPackEntry, ContentPackManifest, - findIndexPatterns, + findConfiguration, isIndexPlaceholder, } from '@kbn/content-packs-schema'; import { @@ -88,7 +88,7 @@ export function ExportContentPackFlyout({ }); const indexPatterns = uniq( - contentPack.entries.flatMap((object) => findIndexPatterns(object)) + contentPack.entries.flatMap((object) => findConfiguration(object).patterns) ).filter((index) => !isIndexPlaceholder(index)); setManifest({ From a38af550b88aeb90239e6e9b1d8c887c2cbf5aa0 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 16:40:51 +0200 Subject: [PATCH 07/12] soclient not needed --- .../shared/streams/server/lib/content/content_client.ts | 4 +--- .../shared/streams/server/lib/content/content_service.ts | 5 ++--- x-pack/platform/plugins/shared/streams/server/plugin.ts | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts index 5ff4c1de3a23f..5828942e208c2 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/content_client.ts @@ -8,7 +8,6 @@ import { ContentPackSavedObjectLinks, SavedObjectLinkWithReferences, } from '@kbn/content-packs-schema'; -import { SavedObjectsClientContract } from '@kbn/core/server'; import { IStorageClient, IndexStorageSettings, types } from '@kbn/storage-adapter'; import objectHash from 'object-hash'; import { CONTENT_NAME, STREAM_NAME } from './fields'; @@ -41,7 +40,6 @@ export class ContentClient { constructor( private readonly clients: { storageClient: IStorageClient; - soClient: SavedObjectsClientContract; } ) {} @@ -56,7 +54,7 @@ export class ContentClient { }, }); - return response; + return response.hits.hits.map((hit) => hit._source); } async getStoredContentPack(streamName: string, contentName: string) { diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts index 8a88b3d6dda9c..5564624f18758 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/content_service.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import { CoreSetup, Logger } from '@kbn/core/server'; import { StorageIndexAdapter } from '@kbn/storage-adapter'; import { StreamsPluginStartDependencies } from '../../types'; import { @@ -21,7 +21,7 @@ export class ContentService { private readonly logger: Logger ) {} - async getClientWithRequest({ request }: { request: KibanaRequest }): Promise { + async getClient(): Promise { const [coreStart] = await this.coreSetup.getStartServices(); const adapter = new StorageIndexAdapter( @@ -32,7 +32,6 @@ export class ContentService { return new ContentClient({ storageClient: adapter.getClient(), - soClient: coreStart.savedObjects.getScopedClient(request), }); } } diff --git a/x-pack/platform/plugins/shared/streams/server/plugin.ts b/x-pack/platform/plugins/shared/streams/server/plugin.ts index 0af7acac066c1..3b448e5c84639 100644 --- a/x-pack/platform/plugins/shared/streams/server/plugin.ts +++ b/x-pack/platform/plugins/shared/streams/server/plugin.ts @@ -88,7 +88,7 @@ export class StreamsPlugin const [[coreStart, pluginsStart], assetClient, contentClient] = await Promise.all([ core.getStartServices(), assetService.getClientWithRequest({ request }), - contentService.getClientWithRequest({ request }), + contentService.getClient(), ]); const streamsClient = await streamsService.getClientWithRequest({ request, assetClient }); From 58abbaeb441fa2ce7e36745658f37e3261b4cce9 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 18:16:43 +0200 Subject: [PATCH 08/12] ensure stream exists in preview endpoint --- .../plugins/shared/streams/server/routes/content/route.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts index b6300c3b5c4f1..0e4876b6c2b20 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts @@ -237,10 +237,14 @@ const previewContentRoute = createServerRoute({ security: { authz: { enabled: false, - reason: 'This API does not use any user credentials.', + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, - async handler({ params }) { + async handler({ request, params, getScopedClients }): Promise { + const { streamsClient } = await getScopedClients({ request }); + await streamsClient.ensureStream(params.path.name); + return await parseArchive(params.body.content); }, }); From 2ae4c8dc538a8bbe52c82760ecb6df840208cdf5 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 22 Apr 2025 19:39:11 +0200 Subject: [PATCH 09/12] tests --- .../server/lib/content/saved_object.test.ts | 68 +++++++++++++++++++ .../server/lib/content/saved_object.ts | 5 +- .../streams/server/routes/content/route.ts | 1 + 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts new file mode 100644 index 0000000000000..128369bda85a8 --- /dev/null +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from 'expect'; +import { ContentPackSavedObject, ContentPackSavedObjectLinks } from '@kbn/content-packs-schema'; +import { savedObjectLinks } from './saved_object'; + +describe('Saved object helpers', () => { + describe('savedObjectLinks', () => { + it('reuses existing link', () => { + const existingLinks = { + dashboards: [ + { + source_id: 'foo', + target_id: 'foo-copy', + references: [{ source_id: 'index1', target_id: 'index1-copy' }], + }, + ], + } as ContentPackSavedObjectLinks; + + const links = savedObjectLinks( + [ + { type: 'dashboard', id: 'foo', references: [{ type: 'index-pattern', id: 'index1' }] }, + ] as ContentPackSavedObject[], + existingLinks + ); + + expect(links).toEqual({ + dashboards: [ + { + source_id: 'foo', + target_id: 'foo-copy', + references: [{ source_id: 'index1', target_id: 'index1-copy' }], + }, + ], + }); + }); + + it('generates new ids when no existing links', () => { + const existingLinks = { dashboards: [] } as ContentPackSavedObjectLinks; + + const links = savedObjectLinks( + [ + { type: 'dashboard', id: 'foo', references: [{ type: 'index-pattern', id: 'index1' }] }, + ] as ContentPackSavedObject[], + existingLinks + ); + + const expectUuid = expect.stringMatching( + /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/ + ); + + expect(links).toEqual({ + dashboards: [ + { + source_id: 'foo', + target_id: expectUuid, + references: [{ source_id: 'index1', target_id: expectUuid }], + }, + ], + }); + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index 6816123f049c1..bacf2664dd2cb 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -91,10 +91,7 @@ export function prepareForImport({ return updateIds(uniqObjects, links); } -export function updateIds( - savedObjects: ContentPackSavedObject[], - links: ContentPackSavedObjectLinks -) { +function updateIds(savedObjects: ContentPackSavedObject[], links: ContentPackSavedObjectLinks) { const existingLinks = links.dashboards.flatMap((ref) => [ref, ...ref.references]); const targetId = (sourceId: string) => { const link = existingLinks.find(({ source_id: id }) => id === sourceId); diff --git a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts index 0e4876b6c2b20..a72a4418ae090 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts @@ -11,6 +11,7 @@ import { z } from '@kbn/zod'; import { createConcatStream, createListStream, createPromiseFromStreams } from '@kbn/utils'; import { installManagedIndexPattern } from '@kbn/fleet-plugin/server/services/epm/kibana/assets/install'; import { + ContentPack, ContentPackSavedObject, contentPackIncludedObjectsSchema, isIncludeAll, From 218b23d062a5832fae99e0ea5d8d4a299e74347e Mon Sep 17 00:00:00 2001 From: klacabane Date: Sat, 26 Apr 2025 07:27:23 +0200 Subject: [PATCH 10/12] dedup reference links --- .../server/lib/content/saved_object.test.ts | 32 +++++++++++++++++++ .../server/lib/content/saved_object.ts | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts index 128369bda85a8..c7aafa27abdbe 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts @@ -64,5 +64,37 @@ describe('Saved object helpers', () => { ], }); }); + + it('generates a unique id for duplicated references', () => { + const existingLinks = { dashboards: [] } as ContentPackSavedObjectLinks; + + const links = savedObjectLinks( + [ + { + type: 'dashboard', + id: 'foo', + references: [ + { type: 'index-pattern', id: 'index1' }, + { type: 'index-pattern', id: 'index1' }, + ], + }, + ] as ContentPackSavedObject[], + existingLinks + ); + + const expectUuid = expect.stringMatching( + /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/ + ); + + expect(links).toEqual({ + dashboards: [ + { + source_id: 'foo', + target_id: expectUuid, + references: [{ source_id: 'index1', target_id: expectUuid }], + }, + ], + }); + }); }); }); diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index bacf2664dd2cb..90c2334098106 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -133,7 +133,7 @@ export function savedObjectLinks( return { source_id: object.id, target_id: existingLink?.target_id ?? v4(), - references: object.references.map((ref) => ({ + references: uniqBy(object.references, (ref) => ref.id).map((ref) => ({ source_id: ref.id, target_id: existingLink?.references.find((existingRef) => ref.id === existingRef.source_id) From 3974efef216c702a1bfc8f3cc17ee8a88b9db875 Mon Sep 17 00:00:00 2001 From: klacabane Date: Sat, 26 Apr 2025 07:55:06 +0200 Subject: [PATCH 11/12] dont generate link for non resolved references --- .../server/lib/content/saved_object.test.ts | 32 +++++++++++++++++++ .../server/lib/content/saved_object.ts | 15 +++++---- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts index c7aafa27abdbe..7a07a24b029f3 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.test.ts @@ -25,6 +25,7 @@ describe('Saved object helpers', () => { const links = savedObjectLinks( [ { type: 'dashboard', id: 'foo', references: [{ type: 'index-pattern', id: 'index1' }] }, + { type: 'index-pattern', id: 'index1' }, ] as ContentPackSavedObject[], existingLinks ); @@ -46,6 +47,7 @@ describe('Saved object helpers', () => { const links = savedObjectLinks( [ { type: 'dashboard', id: 'foo', references: [{ type: 'index-pattern', id: 'index1' }] }, + { type: 'index-pattern', id: 'index1' }, ] as ContentPackSavedObject[], existingLinks ); @@ -78,6 +80,7 @@ describe('Saved object helpers', () => { { type: 'index-pattern', id: 'index1' }, ], }, + { type: 'index-pattern', id: 'index1' }, ] as ContentPackSavedObject[], existingLinks ); @@ -96,5 +99,34 @@ describe('Saved object helpers', () => { ], }); }); + + it('does not generate a link for references not included', () => { + const existingLinks = { dashboards: [] } as ContentPackSavedObjectLinks; + + const links = savedObjectLinks( + [ + { + type: 'dashboard', + id: 'foo', + references: [{ type: 'index-pattern', id: 'index1' }], + }, + ] as ContentPackSavedObject[], + existingLinks + ); + + const expectUuid = expect.stringMatching( + /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/ + ); + + expect(links).toEqual({ + dashboards: [ + { + source_id: 'foo', + target_id: expectUuid, + references: [], + }, + ], + }); + }); }); }); diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index 90c2334098106..ba152849869e0 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -133,12 +133,15 @@ export function savedObjectLinks( return { source_id: object.id, target_id: existingLink?.target_id ?? v4(), - references: uniqBy(object.references, (ref) => ref.id).map((ref) => ({ - source_id: ref.id, - target_id: - existingLink?.references.find((existingRef) => ref.id === existingRef.source_id) - ?.target_id ?? v4(), - })), + references: uniqBy(object.references, (ref) => ref.id) + // do not generate links for references not included in the content pack + .filter((ref) => savedObjects.find((so) => so.id === ref.id)) + .map((ref) => ({ + source_id: ref.id, + target_id: + existingLink?.references.find((existingRef) => ref.id === existingRef.source_id) + ?.target_id ?? v4(), + })), }; }); From f4293ffe94678fb7375a8afc8c39c19973ac60e7 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 29 Apr 2025 15:04:27 +0200 Subject: [PATCH 12/12] support lens references --- .../src/helpers/index_pattern.ts | 10 +-- .../src/models/saved_object.ts | 22 +++++- .../streams/server/lib/content/archive.ts | 13 ++-- .../server/lib/content/saved_object.ts | 69 ++++++++++--------- .../streams/server/routes/content/route.ts | 4 +- 5 files changed, 70 insertions(+), 48 deletions(-) diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts index 2ccd5ae36cbc9..8659b1f759d94 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/helpers/index_pattern.ts @@ -95,11 +95,9 @@ export function replaceIndexPatterns( } function locateConfiguration( - object: ContentPackSavedObject, + content: ContentPackSavedObject, options: TraverseOptions ): ContentPackSavedObject { - const content = object; - if (content.type === 'index-pattern') { content.attributes = options.indexPattern(content.attributes); } @@ -113,7 +111,11 @@ function locateConfiguration( attributes.panelsJSON = JSON.stringify(panels); } - return object; + if (content.type === 'lens') { + content.attributes = traverseLensPanel(content.attributes, options); + } + + return content; } function traversePanel(panel: SavedDashboardPanel, options: TraverseOptions) { diff --git a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts index 45a1ed59e7dd3..2ad19bdecfd02 100644 --- a/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts +++ b/x-pack/platform/packages/shared/kbn-content-packs-schema/src/models/saved_object.ts @@ -5,13 +5,33 @@ * 2.0. */ +import path from 'path'; import type { SavedObject } from '@kbn/core/server'; import type { DashboardAttributes } from '@kbn/dashboard-plugin/common/content_management/v2'; import type { DataViewSavedObjectAttrs } from '@kbn/data-views-plugin/common/data_views'; +import type { LensAttributes } from '@kbn/lens-embeddable-utils'; + +export const SUPPORTED_SAVED_OBJECT_TYPES = [ + { type: 'dashboard', dir: 'dashboard' }, + { type: 'index-pattern', dir: 'index_pattern' }, + { type: 'lens', dir: 'lens' }, +]; +export const isSupportedSavedObjectType = ( + entry: SavedObject +): entry is ContentPackSavedObject => { + return SUPPORTED_SAVED_OBJECT_TYPES.some(({ type }) => type === entry.type); +}; + +export const isSupportedSavedObjectFile = (filepath: string) => { + return SUPPORTED_SAVED_OBJECT_TYPES.some( + ({ dir }) => path.dirname(filepath) === path.join('kibana', dir) + ); +}; type ContentPackDashboard = SavedObject & { type: 'dashboard' }; type ContentPackDataView = SavedObject & { type: 'index-pattern' }; -export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView; +type ContentPackLens = SavedObject & { type: 'lens' }; +export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView | ContentPackLens; export interface SavedObjectLink { source_id: string; diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/archive.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/archive.ts index 54545358b94e7..5cc63fdecf9a2 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/archive.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/archive.ts @@ -10,7 +10,10 @@ import { ContentPack, ContentPackEntry, ContentPackManifest, + SUPPORTED_SAVED_OBJECT_TYPES, contentPackManifestSchema, + isSupportedSavedObjectFile, + isSupportedSavedObjectType, } from '@kbn/content-packs-schema'; import AdmZip from 'adm-zip'; import path from 'path'; @@ -35,13 +38,9 @@ export async function parseArchive(archive: Readable): Promise { const entries: ContentPackEntry[] = []; zip.forEach((entry) => { const filepath = path.join(...entry.entryName.split(path.sep).slice(1)); - const dirname = path.dirname(filepath); if (filepath === 'manifest.yml') { manifestEntry = entry; - } else if ( - dirname === path.join('kibana', 'dashboard') || - dirname === path.join('kibana', 'index_pattern') - ) { + } else if (isSupportedSavedObjectFile(filepath)) { entries.push(JSON.parse(entry.getData().toString())); } }); @@ -65,8 +64,8 @@ export async function generateArchive(manifest: ContentPackManifest, objects: Co const rootDir = `${manifest.name}-${manifest.version}`; objects.forEach((object: ContentPackEntry) => { - if (object.type === 'dashboard' || object.type === 'index-pattern') { - const dir = object.type === 'dashboard' ? 'dashboard' : 'index_pattern'; + if (isSupportedSavedObjectType(object)) { + const dir = SUPPORTED_SAVED_OBJECT_TYPES.find(({ type }) => type === object.type)!.dir; zip.addFile( path.join(rootDir, 'kibana', dir, `${object.id}.json`), Buffer.from(JSON.stringify(object, null, 2)) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts index ba152849869e0..a5f7b6910e4f8 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/content/saved_object.ts @@ -13,39 +13,38 @@ import { INDEX_PLACEHOLDER, findConfiguration, isIncludeAll, + isSupportedSavedObjectType, replaceIndexPatterns, } from '@kbn/content-packs-schema'; import { compact, uniqBy } from 'lodash'; +import { SavedObject } from '@kbn/core/server'; export function prepareForExport({ savedObjects, source, replacedPatterns = [], }: { - savedObjects: ContentPackSavedObject[]; + savedObjects: SavedObject[]; source: string; replacedPatterns?: string[]; }) { - return savedObjects.map((object) => { - if (object.type === 'dashboard' || object.type === 'index-pattern') { - const { patterns } = findConfiguration(object); - const replacements = { - ...replacedPatterns.reduce((acc, pattern) => { - acc[pattern] = INDEX_PLACEHOLDER; + return savedObjects.filter(isSupportedSavedObjectType).map((object) => { + const { patterns } = findConfiguration(object); + const replacements = { + ...replacedPatterns.reduce((acc, pattern) => { + acc[pattern] = INDEX_PLACEHOLDER; + return acc; + }, {} as Record), + + ...patterns + .filter((pattern) => pattern.startsWith(source)) + .reduce((acc, pattern) => { + acc[pattern] = pattern.replace(source, INDEX_PLACEHOLDER); return acc; }, {} as Record), + }; - ...patterns - .filter((pattern) => pattern.startsWith(source)) - .reduce((acc, pattern) => { - acc[pattern] = pattern.replace(source, INDEX_PLACEHOLDER); - return acc; - }, {} as Record), - }; - - return replaceIndexPatterns(object, replacements); - } - return object; + return replaceIndexPatterns(object, replacements); }); } @@ -55,7 +54,7 @@ export function prepareForImport({ target, links, }: { - savedObjects: ContentPackSavedObject[]; + savedObjects: SavedObject[]; include: ContentPackIncludedObjects; target: string; links: ContentPackSavedObjectLinks; @@ -76,40 +75,42 @@ export function prepareForImport({ ), ]), ({ id }) => id - ).map((object) => { - const { patterns } = findConfiguration(object); - const replacements = patterns - .filter((pattern) => pattern.startsWith(INDEX_PLACEHOLDER)) - .reduce((acc, pattern) => { - acc[pattern] = pattern.replace(INDEX_PLACEHOLDER, target); - return acc; - }, {} as Record); + ) + .filter(isSupportedSavedObjectType) + .map((object) => { + const { patterns } = findConfiguration(object); + const replacements = patterns + .filter((pattern) => pattern.startsWith(INDEX_PLACEHOLDER)) + .reduce((acc, pattern) => { + acc[pattern] = pattern.replace(INDEX_PLACEHOLDER, target); + return acc; + }, {} as Record); - return replaceIndexPatterns(object, replacements); - }); + return replaceIndexPatterns(object, replacements); + }); return updateIds(uniqObjects, links); } function updateIds(savedObjects: ContentPackSavedObject[], links: ContentPackSavedObjectLinks) { const existingLinks = links.dashboards.flatMap((ref) => [ref, ...ref.references]); - const targetId = (sourceId: string) => { - const link = existingLinks.find(({ source_id: id }) => id === sourceId); + const targetId = ({ id, type }: { id: string; type: string }) => { + const link = existingLinks.find(({ source_id: sourceId }) => sourceId === id); if (!link) { - throw new Error(`link for [${sourceId}] was not generated`); + throw new Error(`link for object [type: ${type} | id: ${id}] was not generated`); } return link.target_id; }; savedObjects.forEach((object) => { - object.id = targetId(object.id); + object.id = targetId(object); object.references.forEach((ref) => { // only update the id if the reference is included in the content pack. // a missing reference is not necessarily an error condition since it could // point to a pre existing saved object, for example logs-* and metrics-* // data views if (savedObjects.find((so) => so.id === ref.id)) { - ref.id = targetId(ref.id); + ref.id = targetId(ref); } }); }); diff --git a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts index 82c8c1e424ba9..4671e81338895 100644 --- a/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts +++ b/x-pack/platform/plugins/shared/streams/server/routes/content/route.ts @@ -12,10 +12,10 @@ import { createConcatStream, createListStream, createPromiseFromStreams } from ' import { installManagedIndexPattern } from '@kbn/fleet-plugin/server/services/epm/kibana/assets/install'; import { ContentPack, - ContentPackSavedObject, contentPackIncludedObjectsSchema, isIncludeAll, } from '@kbn/content-packs-schema'; +import type { SavedObject } from '@kbn/core/server'; import { STREAMS_API_PRIVILEGES } from '../../../common/constants'; import { Asset } from '../../../common'; import { DashboardAsset, DashboardLink } from '../../../common/assets'; @@ -91,7 +91,7 @@ const exportContentRoute = createServerRoute({ includeReferencesDeep: true, }); - const savedObjects: ContentPackSavedObject[] = await createPromiseFromStreams([ + const savedObjects: SavedObject[] = await createPromiseFromStreams([ exportStream, createConcatStream([]), ]);