From 5fe10785ba2bf7adeb221303c2c55b3c7f841825 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 31 Oct 2024 15:21:53 -0600 Subject: [PATCH 01/18] Initial setup for streams plugin --- package.json | 1 + tsconfig.base.json | 2 + x-pack/plugins/streams/README.md | 3 + x-pack/plugins/streams/common/config.ts | 30 + x-pack/plugins/streams/common/constants.ts | 9 + x-pack/plugins/streams/common/types.ts | 123 ++ x-pack/plugins/streams/jest.config.js | 18 + x-pack/plugins/streams/kibana.jsonc | 29 + x-pack/plugins/streams/public/index.ts | 13 + x-pack/plugins/streams/public/plugin.ts | 32 + x-pack/plugins/streams/public/types.ts | 16 + x-pack/plugins/streams/server/index.ts | 19 + .../lib/streams/bootstrap_root_assets.ts | 30 + .../server/lib/streams/bootstrap_stream.ts | 62 + .../component_templates/generate_layer.ts | 36 + .../component_templates/logs_all_layer.ts | 1186 +++++++++++++++++ .../errors/component_template_not_found.ts | 13 + .../streams/errors/definition_id_invalid.ts | 13 + .../streams/errors/definition_not_found.ts | 13 + .../streams/errors/fork_condition_missing.ts | 13 + .../lib/streams/errors/id_conflict_error.ts | 18 + .../server/lib/streams/errors/index.ts | 15 + .../errors/index_template_not_found.ts | 13 + .../errors/ingest_pipeline_not_found.ts | 13 + .../lib/streams/errors/permission_denied.ts | 13 + .../lib/streams/errors/security_exception.ts | 13 + .../server/lib/streams/helpers/retry.ts | 58 + .../generate_index_template.ts | 38 + .../lib/streams/index_templates/logs_all.ts | 12 + .../generate_ingest_pipeline.ts | 33 + .../generate_reroute_pipeline.ts | 42 + .../logs_all_default_pipeline.ts | 51 + .../logs_all_json_pipeline.ts | 58 + .../lib/streams/root_stream_definition.ts | 14 + .../streams/server/lib/streams/stream_crud.ts | 109 ++ x-pack/plugins/streams/server/plugin.ts | 130 ++ .../server/routes/create_server_route.ts | 11 + x-pack/plugins/streams/server/routes/index.ts | 18 + .../streams/server/routes/streams/enable.ts | 45 + .../streams/server/routes/streams/fork.ts | 74 + .../streams/server/routes/streams/read.ts | 41 + x-pack/plugins/streams/server/routes/types.ts | 22 + .../server/templates/components/base.ts | 58 + .../templates/manage_index_templates.ts | 107 ++ .../templates/streams_index_template.ts | 43 + x-pack/plugins/streams/server/types.ts | 47 + x-pack/plugins/streams/tsconfig.json | 41 + yarn.lock | 4 + 48 files changed, 2802 insertions(+) create mode 100644 x-pack/plugins/streams/README.md create mode 100644 x-pack/plugins/streams/common/config.ts create mode 100644 x-pack/plugins/streams/common/constants.ts create mode 100644 x-pack/plugins/streams/common/types.ts create mode 100644 x-pack/plugins/streams/jest.config.js create mode 100644 x-pack/plugins/streams/kibana.jsonc create mode 100644 x-pack/plugins/streams/public/index.ts create mode 100644 x-pack/plugins/streams/public/plugin.ts create mode 100644 x-pack/plugins/streams/public/types.ts create mode 100644 x-pack/plugins/streams/server/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/retry.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/stream_crud.ts create mode 100644 x-pack/plugins/streams/server/plugin.ts create mode 100644 x-pack/plugins/streams/server/routes/create_server_route.ts create mode 100644 x-pack/plugins/streams/server/routes/index.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/enable.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/fork.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/read.ts create mode 100644 x-pack/plugins/streams/server/routes/types.ts create mode 100644 x-pack/plugins/streams/server/templates/components/base.ts create mode 100644 x-pack/plugins/streams/server/templates/manage_index_templates.ts create mode 100644 x-pack/plugins/streams/server/templates/streams_index_template.ts create mode 100644 x-pack/plugins/streams/server/types.ts create mode 100644 x-pack/plugins/streams/tsconfig.json diff --git a/package.json b/package.json index 4a91266058ccb..4054d80f42312 100644 --- a/package.json +++ b/package.json @@ -928,6 +928,7 @@ "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a", "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b", "@kbn/std": "link:packages/kbn-std", + "@kbn/streams-plugin": "link:x-pack/plugins/streams", "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics", "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location", "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture", diff --git a/tsconfig.base.json b/tsconfig.base.json index 727cb930bc606..b06a2ab957a8b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1826,6 +1826,8 @@ "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"], "@kbn/storybook": ["packages/kbn-storybook"], "@kbn/storybook/*": ["packages/kbn-storybook/*"], + "@kbn/streams-plugin": ["x-pack/plugins/streams"], + "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"], "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"], "@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"], "@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"], diff --git a/x-pack/plugins/streams/README.md b/x-pack/plugins/streams/README.md new file mode 100644 index 0000000000000..9a3539a33535d --- /dev/null +++ b/x-pack/plugins/streams/README.md @@ -0,0 +1,3 @@ +# Streams Plugin + +This plugin provides an interface to manage streams \ No newline at end of file diff --git a/x-pack/plugins/streams/common/config.ts b/x-pack/plugins/streams/common/config.ts new file mode 100644 index 0000000000000..3371b1b6d8cbc --- /dev/null +++ b/x-pack/plugins/streams/common/config.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const configSchema = schema.object({}); + +export type StreamsConfig = TypeOf; + +/** + * The following map is passed to the server plugin setup under the + * exposeToBrowser: option, and controls which of the above config + * keys are allow-listed to be available in the browser config. + * + * NOTE: anything exposed here will be visible in the UI dev tools, + * and therefore MUST NOT be anything that is sensitive information! + */ +export const exposeToBrowserConfig = {} as const; + +type ValidKeys = keyof { + [K in keyof typeof exposeToBrowserConfig as (typeof exposeToBrowserConfig)[K] extends true + ? K + : never]: true; +}; + +export type StreamsPublicConfig = Pick; diff --git a/x-pack/plugins/streams/common/constants.ts b/x-pack/plugins/streams/common/constants.ts new file mode 100644 index 0000000000000..acc66e9286f87 --- /dev/null +++ b/x-pack/plugins/streams/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ASSET_VERSION = 1; +export const STREAMS_INDEX = '.streams'; diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts new file mode 100644 index 0000000000000..a0bf385ed03cd --- /dev/null +++ b/x-pack/plugins/streams/common/types.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; + +export const metricNameSchema = z + .string() + .length(1) + .regex(/[a-zA-Z]/) + .toUpperCase(); + +const apiMetricSchema = z.object({ + name: z.string(), + metrics: z.array( + z.object({ + name: metricNameSchema, + path: z.string(), + }) + ), + equation: z.string(), +}); + +const metaDataSchemaObj = z.object({ + source: z.string(), + destination: z.string(), + fromRoot: z.boolean().default(false), + expand: z.optional( + z.object({ + regex: z.string(), + map: z.array(z.string()), + }) + ), +}); + +type MetadataSchema = z.infer; + +const metadataSchema = metaDataSchemaObj + .or( + z.string().transform( + (value) => + ({ + source: value, + destination: value, + fromRoot: false, + } as MetadataSchema) + ) + ) + .transform((metadata) => ({ + ...metadata, + destination: metadata.destination ?? metadata.source, + })) + .superRefine((value, ctx) => { + if (value.source.length === 0) { + ctx.addIssue({ + path: ['source'], + code: z.ZodIssueCode.custom, + message: 'source should not be empty', + }); + } + if (value.destination.length === 0) { + ctx.addIssue({ + path: ['destination'], + code: z.ZodIssueCode.custom, + message: 'destination should not be empty', + }); + } + }); + +export const apiScraperDefinitionSchema = z.object({ + id: z.string().regex(/^[\w-]+$/), + name: z.string(), + identityFields: z.array(z.string()), + metadata: z.array(metadataSchema), + metrics: z.array(apiMetricSchema), + source: z.object({ + type: z.literal('elasticsearch_api'), + endpoint: z.string(), + method: z.enum(['GET', 'POST']), + params: z.object({ + body: z.record(z.string(), z.any()), + query: z.record(z.string(), z.any()), + }), + collect: z.object({ + path: z.string(), + keyed: z.boolean().default(false), + }), + }), + managed: z.boolean().default(false), + apiKeyId: z.optional(z.string()), +}); + +export type ApiScraperDefinition = z.infer; + +/** + * Example of a "root" StreamEntity + * { + * "id": "logs-all", + * "type": "logs", + * "dataset": "all", + * } + * + * Example of a forked StreamEntity + * { + * "id": "logs-nginx", + * "type": "logs", + * "dataset": "nginx", + * "forked_from": "logs-all" + * } + */ + +export const streamDefinitonSchema = z.object({ + id: z.string(), + type: z.enum(['logs', 'metrics']), + dataset: z.string(), + forked_from: z.optional(z.string()), + condition: z.optional(z.string()), +}); + +export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/jest.config.js b/x-pack/plugins/streams/jest.config.js new file mode 100644 index 0000000000000..2bf0ab535784d --- /dev/null +++ b/x-pack/plugins/streams/jest.config.js @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/logsai/stream_entities_manager'], + coverageDirectory: + '/target/kibana-coverage/jest/x-pack/plugins/logsai/stream_entities_manager', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/plugins/logsai/stream_entities_manager/{common,public,server}/**/*.{js,ts,tsx}', + ], +}; diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc new file mode 100644 index 0000000000000..d97a0b371bb27 --- /dev/null +++ b/x-pack/plugins/streams/kibana.jsonc @@ -0,0 +1,29 @@ +{ + "type": "plugin", + "id": "@kbn/streams-plugin", + "owner": "@elastic/obs-entities", + "description": "A manager for Streams", + "group": "observability", + "visibility": "private", + "plugin": { + "id": "streams", + "configPath": ["xpack", "streams"], + "browser": true, + "server": true, + "requiredPlugins": [ + "data", + "security", + "encryptedSavedObjects", + "usageCollection", + "licensing", + "taskManager", + "features" + ], + "optionalPlugins": [ + "cloud", + "serverless" + ], + "requiredBundles": [ + ] + } +} diff --git a/x-pack/plugins/streams/public/index.ts b/x-pack/plugins/streams/public/index.ts new file mode 100644 index 0000000000000..5b83ea1d297d3 --- /dev/null +++ b/x-pack/plugins/streams/public/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { Plugin } from './plugin'; + +export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => { + return new Plugin(context); +}; diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts new file mode 100644 index 0000000000000..f35d18e06ff70 --- /dev/null +++ b/x-pack/plugins/streams/public/plugin.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public'; +import { Logger } from '@kbn/logging'; + +import type { StreamsPublicConfig } from '../common/config'; +import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; + +export class Plugin implements StreamsPluginClass { + public config: StreamsPublicConfig; + public logger: Logger; + + constructor(context: PluginInitializerContext<{}>) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + setup(core: CoreSetup, pluginSetup: StreamsPluginSetup) { + return {}; + } + + start(core: CoreStart) { + return {}; + } + + stop() {} +} diff --git a/x-pack/plugins/streams/public/types.ts b/x-pack/plugins/streams/public/types.ts new file mode 100644 index 0000000000000..61e5fa94098f0 --- /dev/null +++ b/x-pack/plugins/streams/public/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Plugin as PluginClass } from '@kbn/core/public'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export type StreamsPluginClass = PluginClass<{}, {}, StreamsPluginSetup, StreamsPluginStart>; diff --git a/x-pack/plugins/streams/server/index.ts b/x-pack/plugins/streams/server/index.ts new file mode 100644 index 0000000000000..bd8aee304ad15 --- /dev/null +++ b/x-pack/plugins/streams/server/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '@kbn/core-plugins-server'; +import { StreamsConfig } from '../common/config'; +import { StreamsPluginSetup, StreamsPluginStart, config } from './plugin'; +import { StreamsRouteRepository } from './routes'; + +export type { StreamsConfig, StreamsPluginSetup, StreamsPluginStart, StreamsRouteRepository }; +export { config }; + +export const plugin = async (context: PluginInitializerContext) => { + const { StreamsPlugin } = await import('./plugin'); + return new StreamsPlugin(context); +}; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts new file mode 100644 index 0000000000000..a2026c2105f37 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { logsAllLayer } from './component_templates/logs_all_layer'; +import { logsAllDefaultPipeline } from './ingest_pipelines/logs_all_default_pipeline'; +import { logsAllIndexTemplate } from './index_templates/logs_all'; +import { logsAllJsonPipeline } from './ingest_pipelines/logs_all_json_pipeline'; + +interface BootstrapRootEntityParams { + esClient: ElasticsearchClient; + logger: Logger; +} + +export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { + await upsertComponent({ esClient, logger, component: logsAllLayer }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsAllJsonPipeline }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsAllDefaultPipeline }); + await upsertTemplate({ esClient, logger, template: logsAllIndexTemplate }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts new file mode 100644 index 0000000000000..c609af5b49978 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { StreamDefinition } from '../../../common/types'; +import { generateLayer } from './component_templates/generate_layer'; +import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { generateIndexTemplate } from './index_templates/generate_index_template'; +import { getIndexTemplateComponents } from './stream_crud'; +import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; + +interface BootstrapStreamParams { + scopedClusterClient: IScopedClusterClient; + definition: StreamDefinition; + rootDefinition: StreamDefinition; + logger: Logger; +} +export async function bootstrapStream({ + scopedClusterClient, + definition, + rootDefinition, + logger, +}: BootstrapStreamParams) { + const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ + scopedClusterClient, + definition: rootDefinition, + }); + const reroutePipeline = await generateReroutePipeline({ + esClient: scopedClusterClient.asCurrentUser, + definition: rootDefinition, + }); + await upsertComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + component: generateLayer(definition.id), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: generateIngestPipeline(definition.id), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: reroutePipeline, + }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), + }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts new file mode 100644 index 0000000000000..bd8ee24be4dc1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateLayer(id: string): ClusterPutComponentTemplateRequest { + return { + name: `${id}@layer`, + template: { + settings: { + index: { + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, + }, + ignore_malformed: true, + }, + }, + }, + }, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `Default settings for the ${id} StreamEntity`, + }, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts new file mode 100644 index 0000000000000..33e43bce29544 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts @@ -0,0 +1,1186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllLayer: ClusterPutComponentTemplateRequest = { + name: 'logs-all@layer', + template: { + settings: { + index: { + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, + }, + ignore_malformed: true, + }, + }, + }, + mappings: { + dynamic: false, + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + }, + 'data_stream.namespace': { + type: 'constant_keyword', + }, + 'data_stream.dataset': { + type: 'constant_keyword', + value: 'all', + }, + 'data_stream.type': { + type: 'constant_keyword', + value: 'logs', + }, + + // Base + labels: { + type: 'object', + }, + message: { + type: 'match_only_text', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + + // file + file: { + properties: { + accessed: { + type: 'date', + }, + attributes: { + ignore_above: 1024, + type: 'keyword', + }, + code_signature: { + properties: { + digest_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + exists: { + type: 'boolean', + }, + signing_id: { + ignore_above: 1024, + type: 'keyword', + }, + status: { + ignore_above: 1024, + type: 'keyword', + }, + subject_name: { + ignore_above: 1024, + type: 'keyword', + }, + team_id: { + ignore_above: 1024, + type: 'keyword', + }, + timestamp: { + type: 'date', + }, + trusted: { + type: 'boolean', + }, + valid: { + type: 'boolean', + }, + }, + }, + created: { + type: 'date', + }, + ctime: { + type: 'date', + }, + device: { + ignore_above: 1024, + type: 'keyword', + }, + directory: { + ignore_above: 1024, + type: 'keyword', + }, + drive_letter: { + ignore_above: 1, + type: 'keyword', + }, + elf: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + byte_order: { + ignore_above: 1024, + type: 'keyword', + }, + cpu_type: { + ignore_above: 1024, + type: 'keyword', + }, + creation_date: { + type: 'date', + }, + exports: { + type: 'flattened', + }, + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + header: { + properties: { + abi_version: { + ignore_above: 1024, + type: 'keyword', + }, + class: { + ignore_above: 1024, + type: 'keyword', + }, + data: { + ignore_above: 1024, + type: 'keyword', + }, + entrypoint: { + type: 'long', + }, + object_version: { + ignore_above: 1024, + type: 'keyword', + }, + os_abi: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + sections: { + properties: { + chi2: { + type: 'long', + }, + entropy: { + type: 'long', + }, + flags: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_offset: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + var_entropy: { + type: 'long', + }, + virtual_address: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + segments: { + properties: { + sections: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + shared_libraries: { + ignore_above: 1024, + type: 'keyword', + }, + telfhash: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + extension: { + ignore_above: 1024, + type: 'keyword', + }, + fork_name: { + ignore_above: 1024, + type: 'keyword', + }, + gid: { + ignore_above: 1024, + type: 'keyword', + }, + group: { + ignore_above: 1024, + type: 'keyword', + }, + hash: { + properties: { + md5: { + ignore_above: 1024, + type: 'keyword', + }, + sha1: { + ignore_above: 1024, + type: 'keyword', + }, + sha256: { + ignore_above: 1024, + type: 'keyword', + }, + sha384: { + ignore_above: 1024, + type: 'keyword', + }, + sha512: { + ignore_above: 1024, + type: 'keyword', + }, + ssdeep: { + ignore_above: 1024, + type: 'keyword', + }, + tlsh: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + inode: { + ignore_above: 1024, + type: 'keyword', + }, + macho: { + properties: { + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + sections: { + properties: { + entropy: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + var_entropy: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + symhash: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + mime_type: { + ignore_above: 1024, + type: 'keyword', + }, + mode: { + ignore_above: 1024, + type: 'keyword', + }, + mtime: { + type: 'date', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + owner: { + ignore_above: 1024, + type: 'keyword', + }, + path: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + pe: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + company: { + ignore_above: 1024, + type: 'keyword', + }, + description: { + ignore_above: 1024, + type: 'keyword', + }, + file_version: { + ignore_above: 1024, + type: 'keyword', + }, + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + imphash: { + ignore_above: 1024, + type: 'keyword', + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + original_file_name: { + ignore_above: 1024, + type: 'keyword', + }, + pehash: { + ignore_above: 1024, + type: 'keyword', + }, + product: { + ignore_above: 1024, + type: 'keyword', + }, + sections: { + properties: { + entropy: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + var_entropy: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + }, + }, + size: { + type: 'long', + }, + target_path: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + uid: { + ignore_above: 1024, + type: 'keyword', + }, + x509: { + properties: { + alternative_names: { + ignore_above: 1024, + type: 'keyword', + }, + issuer: { + properties: { + common_name: { + ignore_above: 1024, + type: 'keyword', + }, + country: { + ignore_above: 1024, + type: 'keyword', + }, + distinguished_name: { + ignore_above: 1024, + type: 'keyword', + }, + locality: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + organizational_unit: { + ignore_above: 1024, + type: 'keyword', + }, + state_or_province: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + not_after: { + type: 'date', + }, + not_before: { + type: 'date', + }, + public_key_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + public_key_curve: { + ignore_above: 1024, + type: 'keyword', + }, + public_key_exponent: { + doc_values: false, + index: false, + type: 'long', + }, + public_key_size: { + type: 'long', + }, + serial_number: { + ignore_above: 1024, + type: 'keyword', + }, + signature_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + subject: { + properties: { + common_name: { + ignore_above: 1024, + type: 'keyword', + }, + country: { + ignore_above: 1024, + type: 'keyword', + }, + distinguished_name: { + ignore_above: 1024, + type: 'keyword', + }, + locality: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + organizational_unit: { + ignore_above: 1024, + type: 'keyword', + }, + state_or_province: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + version_number: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + + // Host + host: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + cpu: { + properties: { + usage: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + disk: { + properties: { + read: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + write: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + }, + }, + domain: { + ignore_above: 1024, + type: 'keyword', + }, + geo: { + properties: { + city_name: { + ignore_above: 1024, + type: 'keyword', + }, + continent_code: { + ignore_above: 1024, + type: 'keyword', + }, + continent_name: { + ignore_above: 1024, + type: 'keyword', + }, + country_iso_code: { + ignore_above: 1024, + type: 'keyword', + }, + country_name: { + ignore_above: 1024, + type: 'keyword', + }, + location: { + type: 'geo_point', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + postal_code: { + ignore_above: 1024, + type: 'keyword', + }, + region_iso_code: { + ignore_above: 1024, + type: 'keyword', + }, + region_name: { + ignore_above: 1024, + type: 'keyword', + }, + timezone: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + hostname: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + ip: { + type: 'ip', + }, + mac: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + network: { + properties: { + egress: { + properties: { + bytes: { + type: 'long', + }, + packets: { + type: 'long', + }, + }, + }, + ingress: { + properties: { + bytes: { + type: 'long', + }, + packets: { + type: 'long', + }, + }, + }, + }, + }, + os: { + properties: { + family: { + ignore_above: 1024, + type: 'keyword', + }, + full: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + kernel: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + platform: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + uptime: { + type: 'long', + }, + }, + }, + + // Orchestrator + orchestrator: { + properties: { + api_version: { + ignore_above: 1024, + type: 'keyword', + }, + cluster: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + url: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + namespace: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + resource: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + log: { + properties: { + file: { + properties: { + path: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + level: { + ignore_above: 1024, + type: 'keyword', + }, + logger: { + ignore_above: 1024, + type: 'keyword', + }, + origin: { + properties: { + file: { + properties: { + line: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + function: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + syslog: { + properties: { + facility: { + properties: { + code: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + priority: { + type: 'long', + }, + severity: { + properties: { + code: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + type: 'object', + }, + }, + }, + + // ECS + ecs: { + properties: { + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + // Agent + agent: { + properties: { + build: { + properties: { + original: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + ephemeral_id: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + // Cloud + cloud: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + origin: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + target: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: ASSET_VERSION, + _meta: { + managed: true, + description: 'Default layer for logs-all StreamEntity', + }, + deprecated: false, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts new file mode 100644 index 0000000000000..a7e9cebf98507 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ComponentTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'ComponentTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts new file mode 100644 index 0000000000000..817e8f67bf25d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionIdInvalid extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionIdInvalid'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts new file mode 100644 index 0000000000000..f7e60193baa5f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts new file mode 100644 index 0000000000000..713751dbe4363 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ForkConditionMissing extends Error { + constructor(message: string) { + super(message); + this.name = 'ForkConditionMissing'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts new file mode 100644 index 0000000000000..b2e23e03dd7ef --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApiScraperDefinition } from '../../../../common/types'; + +export class IdConflict extends Error { + public definition: ApiScraperDefinition; + + constructor(message: string, def: ApiScraperDefinition) { + super(message); + this.name = 'IdConflict'; + this.definition = def; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index.ts b/x-pack/plugins/streams/server/lib/streams/errors/index.ts new file mode 100644 index 0000000000000..73842ef3018fe --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './definition_id_invalid'; +export * from './definition_not_found'; +export * from './id_conflict_error'; +export * from './permission_denied'; +export * from './security_exception'; +export * from './index_template_not_found'; +export * from './fork_condition_missing'; +export * from './component_template_not_found'; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts new file mode 100644 index 0000000000000..4f4735dd15fa1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IndexTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IndexTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts new file mode 100644 index 0000000000000..8bf9bbd4933ce --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IngestPipelineNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IngestPipelineNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts new file mode 100644 index 0000000000000..f0133e28063ca --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class PermissionDenied extends Error { + constructor(message: string) { + super(message); + this.name = 'PermissionDenied'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts new file mode 100644 index 0000000000000..0b4ae450c2530 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class SecurityException extends Error { + constructor(message: string) { + super(message); + this.name = 'SecurityException'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts new file mode 100644 index 0000000000000..32604a22bf9be --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setTimeout } from 'timers/promises'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import type { Logger } from '@kbn/logging'; +import { SecurityException } from '../errors'; + +const MAX_ATTEMPTS = 5; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: any) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +/** + * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. + * Should only be used to wrap operations that are idempotent and can be safely executed more than once. + */ +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger?: Logger; attempt?: number } = {} +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... + + logger?.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + await setTimeout(retryDelaySec * 1000); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + if (e.meta?.body?.error?.type === 'security_exception') { + throw new SecurityException(e.meta.body.error.reason); + } + + throw e; + } +}; diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts new file mode 100644 index 0000000000000..5052aacf99b6b --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateIndexTemplate( + id: string, + composedOf: string[] = [], + ignoreMissing: string[] = [] +) { + return { + name: id, + index_patterns: [`${id}-*`], + composed_of: [...composedOf, `${id}@layer`], + priority: 200, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `The index template for ${id} StreamEntity`, + }, + data_stream: { + hidden: false, + }, + template: { + settings: { + index: { + default_pipeline: `${id}@default-pipeline`, + }, + }, + }, + allow_auto_create: true, + ignore_missing_component_templates: [...ignoreMissing, `${id}@layer`], + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts new file mode 100644 index 0000000000000..b04b9b8e6841d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { generateIndexTemplate } from './generate_index_template'; + +export const logsAllIndexTemplate: IndicesPutIndexTemplateRequest = + generateIndexTemplate('logs-all'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts new file mode 100644 index 0000000000000..5411579c285bc --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateIngestPipeline(id: string) { + return { + id: `${id}@default-pipeline`, + processors: [ + { + append: { + field: 'labels.elastic.stream_entities', + value: [id], + }, + }, + { + pipeline: { + name: `${id}@reroutes`, + ignore_missing_pipeline: true, + }, + }, + ], + _meta: { + description: `Default pipeline for the ${id} StreamEntity`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts new file mode 100644 index 0000000000000..a8f0a84ca27d2 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { StreamDefinition } from '../../../../common/types'; +import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; + +interface GenerateReroutePipelineParams { + esClient: ElasticsearchClient; + definition: StreamDefinition; +} + +export async function generateReroutePipeline({ + esClient, + definition, +}: GenerateReroutePipelineParams) { + const response = await esClient.search({ + index: STREAMS_INDEX, + query: { match: { forked_from: definition.id } }, + }); + + return { + id: `${definition.id}@reroutes`, + processors: response.hits.hits.map((doc) => { + return { + reroute: { + dataset: doc._source.dataset, + if: doc._source.condition, + }, + }; + }), + _meta: { + description: `Reoute pipeline for the ${definition.id} StreamEntity`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts new file mode 100644 index 0000000000000..333e752ec9762 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllDefaultPipeline = { + id: 'logs-all@default-pipeline', + processors: [ + { + set: { + description: "If '@timestamp' is missing, set it with the ingest timestamp", + field: '@timestamp', + override: false, + copy_from: '_ingest.timestamp', + }, + }, + { + set: { + field: 'event.ingested', + value: '{{{_ingest.timestamp}}}', + }, + }, + { + append: { + field: 'labels.elastic.stream_entities', + value: ['logs-all'], + }, + }, + { + pipeline: { + name: 'logs-all@json-pipeline', + ignore_missing_pipeline: true, + }, + }, + { + pipeline: { + name: 'logs-all@reroutes', + ignore_missing_pipeline: true, + }, + }, + ], + _meta: { + description: 'Default pipeline for the logs-all StreamEntity', + managed: true, + }, + version: ASSET_VERSION, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts new file mode 100644 index 0000000000000..276b981aafbfd --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllJsonPipeline = { + id: 'logs-all@json-pipeline', + processors: [ + { + rename: { + if: "ctx.message instanceof String && ctx.message.startsWith('{') && ctx.message.endsWith('}')", + field: 'message', + target_field: '_tmp_json_message', + ignore_missing: true, + }, + }, + { + json: { + if: 'ctx._tmp_json_message != null', + field: '_tmp_json_message', + add_to_root: true, + add_to_root_conflict_strategy: 'merge' as const, + allow_duplicate_keys: true, + on_failure: [ + { + rename: { + field: '_tmp_json_message', + target_field: 'message', + ignore_missing: true, + }, + }, + ], + }, + }, + { + dot_expander: { + if: 'ctx._tmp_json_message != null', + field: '*', + override: true, + }, + }, + { + remove: { + field: '_tmp_json_message', + ignore_missing: true, + }, + }, + ], + _meta: { + description: 'automatic parsing of JSON log messages', + managed: true, + }, + version: ASSET_VERSION, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts new file mode 100644 index 0000000000000..1a10a9d5ee807 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../common/types'; + +export const rootStreamDefinition: StreamDefinition = { + id: 'logs-all', + type: 'logs', + dataset: 'all', +}; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts new file mode 100644 index 0000000000000..0294524815d21 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { get } from 'lodash'; +import { StreamDefinition } from '../../../common/types'; +import { STREAMS_INDEX } from '../../../common/constants'; +import { ComponentTemplateNotFound, DefinitionNotFound, IndexTemplateNotFound } from './errors'; + +interface BaseParams { + scopedClusterClient: IScopedClusterClient; +} + +interface BaseParamsWithDefinition extends BaseParams { + definition: StreamDefinition; +} + +export async function createStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { + return scopedClusterClient.asCurrentUser.index({ + id: definition.id, + index: STREAMS_INDEX, + document: definition, + refresh: 'wait_for', + }); +} + +interface ReadStreamParams extends BaseParams { + id: string; +} + +export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { + const response = await scopedClusterClient.asCurrentUser.get({ + id, + index: STREAMS_INDEX, + }); + if (!response.found) { + throw new DefinitionNotFound(`Stream Entity definition for ${id} not found.`); + } + const definition = response._source as StreamDefinition; + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); + const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); + return { + definition, + index_template: indexTemplate, + component_template: componentTemplate, + ingest_pipelines: ingestPipelines, + }; +} + +export async function readIndexTemplate({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ + name: definition.id, + }); + const indexTemplate = response.index_templates.find((doc) => doc.name === definition.id); + if (!indexTemplate) { + throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); + } + return indexTemplate; +} + +export async function readComponentTemplate({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ + name: `${definition.id}@layer`, + }); + const componentTemplate = response.component_templates.find( + (doc) => doc.name === `${definition.id}@layer` + ); + if (!componentTemplate) { + throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); + } + return componentTemplate; +} + +export async function readIngestPipelines({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ + id: `${definition.id}*`, + }); + + return response; +} + +export async function getIndexTemplateComponents({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + return { + composedOf: indexTemplate.index_template.composed_of, + ignoreMissing: get( + indexTemplate, + 'index_template.ignore_missing_component_templates', + [] + ) as string[], + }; +} diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts new file mode 100644 index 0000000000000..16473918e2cfa --- /dev/null +++ b/x-pack/plugins/streams/server/plugin.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CoreSetup, + CoreStart, + DEFAULT_APP_CATEGORIES, + KibanaRequest, + Logger, + Plugin, + PluginConfigDescriptor, + PluginInitializerContext, +} from '@kbn/core/server'; +import { registerRoutes } from '@kbn/server-route-repository'; +import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; +import { installStreamsTemplates } from './templates/manage_index_templates'; +import { StreamsRouteRepository } from './routes'; +import { RouteDependencies } from './routes/types'; +import { + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies, + StreamsServer, +} from './types'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: exposeToBrowserConfig, +}; + +export class StreamsPlugin + implements + Plugin< + StreamsPluginSetup, + StreamsPluginStart, + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies + > +{ + public config: StreamsConfig; + public logger: Logger; + public server?: StreamsServer; + + constructor(context: PluginInitializerContext) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + public setup(core: CoreSetup, plugins: StreamsPluginSetupDependencies): StreamsPluginSetup { + this.server = { + config: this.config, + logger: this.logger, + } as StreamsServer; + + plugins.features.registerKibanaFeature({ + id: 'streams', + name: 'Streams', + order: 1500, + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + category: DEFAULT_APP_CATEGORIES.observability, + privileges: { + all: { + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + savedObject: { + all: [], + read: [], + }, + ui: ['read', 'write'], + api: ['streams_enable', 'streams_fork', 'streams_read'], + }, + read: { + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + api: ['streams_read'], + ui: ['read'], + savedObject: { + all: [], + read: [], + }, + }, + }, + }); + + registerRoutes({ + repository: StreamsRouteRepository, + dependencies: { + server: this.server, + getScopedClients: async ({ request }: { request: KibanaRequest }) => { + const [coreStart] = await core.getStartServices(); + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request); + const soClient = coreStart.savedObjects.getScopedClient(request); + return { scopedClusterClient, soClient }; + }, + }, + core, + logger: this.logger, + }); + + return {}; + } + + public start(core: CoreStart, plugins: StreamsPluginStartDependencies): StreamsPluginStart { + if (this.server) { + this.server.core = core; + this.server.isServerless = core.elasticsearch.getCapabilities().serverless; + this.server.security = plugins.security; + this.server.encryptedSavedObjects = plugins.encryptedSavedObjects; + this.server.taskManager = plugins.taskManager; + } + + const esClient = core.elasticsearch.client.asInternalUser; + installStreamsTemplates({ esClient, logger: this.logger }).catch((err) => + this.logger.error(err) + ); + + return {}; + } + + public stop() {} +} diff --git a/x-pack/plugins/streams/server/routes/create_server_route.ts b/x-pack/plugins/streams/server/routes/create_server_route.ts new file mode 100644 index 0000000000000..94d85a71c82bb --- /dev/null +++ b/x-pack/plugins/streams/server/routes/create_server_route.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createServerRouteFactory } from '@kbn/server-route-repository'; +import { StreamsRouteHandlerResources } from './types'; + +export const createServerRoute = createServerRouteFactory(); diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts new file mode 100644 index 0000000000000..c24362f3ec8a9 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { enableStreamsRoute } from './streams/enable'; +import { forkStreamsRoute } from './streams/fork'; +import { readStreamRoute } from './streams/read'; + +export const StreamsRouteRepository = { + ...enableStreamsRoute, + ...forkStreamsRoute, + ...readStreamRoute, +}; + +export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts new file mode 100644 index 0000000000000..1241434a975df --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { SecurityException } from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { bootstrapRootEntity } from '../../lib/streams/bootstrap_root_assets'; +import { createStream } from '../../lib/streams/stream_crud'; +import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; + +export const enableStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_enable 2023-10-31', + params: z.object({}), + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_enable'], + }, + }, + }, + handler: async ({ request, response, logger, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + await bootstrapRootEntity({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + }); + await createStream({ + scopedClusterClient, + definition: rootStreamDefinition, + }); + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof SecurityException) { + return response.customError({ body: e, statusCode: 400 }); + } + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts new file mode 100644 index 0000000000000..1d21b4194f172 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { streamDefinitonSchema } from '../../../common/types'; +import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; +import { createStream, readStream } from '../../lib/streams/stream_crud'; + +export const forkStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: streamDefinitonSchema, + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + if (!params.body.condition) { + throw new ForkConditionMissing('You must provide a condition to fork a stream'); + } + + const { scopedClusterClient } = await getScopedClients({ request }); + + const { definition: rootDefinition } = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + await createStream({ + scopedClusterClient, + definition: { ...params.body, forked_from: rootDefinition.id }, + }); + + await bootstrapStream({ + scopedClusterClient, + definition: params.body, + rootDefinition, + logger, + }); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if (e instanceof SecurityException || e instanceof ForkConditionMissing) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts new file mode 100644 index 0000000000000..c2f7cfe59b3ec --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { readStream } from '../../lib/streams/stream_crud'; + +export const readStreamRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + params: z.object({ + path: z.object({ id: z.string() }), + }), + handler: async ({ response, params, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const streamEntity = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + return response.ok({ body: streamEntity }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/types.ts b/x-pack/plugins/streams/server/routes/types.ts new file mode 100644 index 0000000000000..d547d56c088cd --- /dev/null +++ b/x-pack/plugins/streams/server/routes/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DefaultRouteHandlerResources } from '@kbn/server-route-repository'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { StreamsServer } from '../types'; + +export interface RouteDependencies { + server: StreamsServer; + getScopedClients: ({ request }: { request: KibanaRequest }) => Promise<{ + scopedClusterClient: IScopedClusterClient; + soClient: SavedObjectsClientContract; + }>; +} + +export type StreamsRouteHandlerResources = RouteDependencies & DefaultRouteHandlerResources; diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts new file mode 100644 index 0000000000000..a4744c6962155 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/components/base.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const STREAMS_BASE_COMPONENT = 'streams@mappings'; + +export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: STREAMS_BASE_COMPONENT, + _meta: { + description: 'Component template for the Stream Entities Manager data set', + managed: true, + }, + template: { + mappings: { + properties: { + labels: { + type: 'object', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + dataset: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + forked_from: { + ignore_above: 1024, + type: 'keyword', + }, + condition: { + ignore_above: 1024, + type: 'keyword', + }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/streams/server/templates/manage_index_templates.ts b/x-pack/plugins/streams/server/templates/manage_index_templates.ts new file mode 100644 index 0000000000000..f562c4dfe4183 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/manage_index_templates.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ClusterPutComponentTemplateRequest, + IndicesPutIndexTemplateRequest, + IngestPutPipelineRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { BaseComponentTemplateConfig } from './components/base'; +import { retryTransientEsErrors } from '../lib/streams/helpers/retry'; +import { streamsIndexTemplate } from './streams_index_template'; + +interface TemplateManagementOptions { + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; + logger: Logger; +} + +interface PipelineManagementOptions { + esClient: ElasticsearchClient; + pipeline: IngestPutPipelineRequest; + logger: Logger; +} + +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + +export const installStreamsTemplates = async ({ + esClient, + logger, +}: { + esClient: ElasticsearchClient; + logger: Logger; +}) => { + await upsertComponent({ + esClient, + logger, + component: BaseComponentTemplateConfig, + }); + await upsertTemplate({ + esClient, + logger, + template: streamsIndexTemplate, + }); +}; + +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function upsertIngestPipeline({ + esClient, + pipeline, + logger, +}: PipelineManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting index template: ${error.message}`); + throw error; + } +} + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); + logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); + } catch (error: any) { + logger.error(`Error updating component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/templates/streams_index_template.ts b/x-pack/plugins/streams/server/templates/streams_index_template.ts new file mode 100644 index 0000000000000..0e843b2289cf3 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/streams_index_template.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { STREAMS_BASE_COMPONENT } from './components/base'; +import { STREAMS_INDEX } from '../../common/constants'; + +export const streamsIndexTemplate: IndicesPutIndexTemplateRequest = { + name: 'stream-entities', + _meta: { + description: + 'Index template for indices managed by the Streams framework for the instance dataset', + ecs_version: '8.0.0', + managed: true, + managed_by: 'streams', + }, + composed_of: [STREAMS_BASE_COMPONENT], + index_patterns: [STREAMS_INDEX], + priority: 200, + template: { + mappings: { + _meta: { + version: '1.6.0', + }, + date_detection: false, + dynamic: false, + }, + settings: { + index: { + codec: 'best_compression', + mapping: { + total_fields: { + limit: 2000, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts new file mode 100644 index 0000000000000..f1cfcb08a2649 --- /dev/null +++ b/x-pack/plugins/streams/server/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart, ElasticsearchClient, Logger } from '@kbn/core/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { + EncryptedSavedObjectsPluginSetup, + EncryptedSavedObjectsPluginStart, +} from '@kbn/encrypted-saved-objects-plugin/server'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { StreamsConfig } from '../common/config'; + +export interface StreamsServer { + core: CoreStart; + config: StreamsConfig; + logger: Logger; + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + isServerless: boolean; + taskManager: TaskManagerStartContract; +} + +export interface ElasticsearchAccessorOptions { + elasticsearchClient: ElasticsearchClient; +} + +export interface StreamsPluginSetupDependencies { + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; + taskManager: TaskManagerSetupContract; + features: FeaturesPluginSetup; +} + +export interface StreamsPluginStartDependencies { + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json new file mode 100644 index 0000000000000..98386b54ca9ea --- /dev/null +++ b/x-pack/plugins/streams/tsconfig.json @@ -0,0 +1,41 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "server/**/*", + "public/**/*", + "types/**/*" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/entities-schema", + "@kbn/config-schema", + "@kbn/core", + "@kbn/server-route-repository-client", + "@kbn/logging", + "@kbn/core-plugins-server", + "@kbn/core-http-server", + "@kbn/security-plugin", + "@kbn/rison", + "@kbn/es-query", + "@kbn/core-elasticsearch-client-server-mocks", + "@kbn/core-saved-objects-api-server-mocks", + "@kbn/logging-mocks", + "@kbn/core-saved-objects-api-server", + "@kbn/core-elasticsearch-server", + "@kbn/task-manager-plugin", + "@kbn/datemath", + "@kbn/server-route-repository", + "@kbn/zod", + "@kbn/zod-helpers", + "@kbn/encrypted-saved-objects-plugin", + "@kbn/licensing-plugin", + "@kbn/alerting-plugin" + ] +} diff --git a/yarn.lock b/yarn.lock index 57ffec1e0a278..e7f4249969902 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6908,6 +6908,10 @@ version "0.0.0" uid "" +"@kbn/streams-plugin@link:x-pack/plugins/streams": + version "0.0.0" + uid "" + "@kbn/synthetics-e2e@link:x-pack/plugins/observability_solution/synthetics/e2e": version "0.0.0" uid "" From 6f940ebda01c9d5df6464591bf60e049594d9022 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 1 Nov 2024 11:21:33 -0600 Subject: [PATCH 02/18] migrating to the streams naming schema; adding an abstraction for the fork conditions --- x-pack/plugins/streams/common/types.ts | 120 ++++------------ x-pack/plugins/streams/jest.config.js | 9 +- .../lib/streams/bootstrap_root_assets.ts | 16 +-- .../component_templates/generate_layer.ts | 4 +- .../{logs_all_layer.ts => logs_layer.ts} | 17 +-- .../lib/streams/errors/id_conflict_error.ts | 7 +- .../lib/streams/errors/malformed_stream_id.ts | 13 ++ .../reroute_condition_to_painless.test.ts | 133 ++++++++++++++++++ .../helpers/reroute_condition_to_painless.ts | 85 +++++++++++ .../generate_index_template.ts | 12 +- .../index_templates/{logs_all.ts => logs.ts} | 3 +- .../generate_ingest_pipeline.ts | 12 +- .../generate_reroute_pipeline.ts | 17 ++- ...t_pipeline.ts => logs_default_pipeline.ts} | 16 +-- ...json_pipeline.ts => logs_json_pipeline.ts} | 4 +- .../lib/streams/root_stream_definition.ts | 5 +- .../streams/server/lib/streams/stream_crud.ts | 48 ++++--- .../streams/server/routes/streams/fork.ts | 13 +- .../streams/server/routes/streams/read.ts | 2 + .../server/templates/components/base.ts | 6 +- 20 files changed, 353 insertions(+), 189 deletions(-) rename x-pack/plugins/streams/server/lib/streams/component_templates/{logs_all_layer.ts => logs_layer.ts} (98%) create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts rename x-pack/plugins/streams/server/lib/streams/index_templates/{logs_all.ts => logs.ts} (79%) rename x-pack/plugins/streams/server/lib/streams/ingest_pipelines/{logs_all_default_pipeline.ts => logs_default_pipeline.ts} (72%) rename x-pack/plugins/streams/server/lib/streams/ingest_pipelines/{logs_all_json_pipeline.ts => logs_json_pipeline.ts} (95%) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index a0bf385ed03cd..5bdc018452376 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -7,117 +7,53 @@ import { z } from '@kbn/zod'; -export const metricNameSchema = z - .string() - .length(1) - .regex(/[a-zA-Z]/) - .toUpperCase(); +const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); -const apiMetricSchema = z.object({ - name: z.string(), - metrics: z.array( - z.object({ - name: metricNameSchema, - path: z.string(), - }) - ), - equation: z.string(), +export const rerouteFilterConditionSchema = z.object({ + field: z.string(), + operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), + value: stringOrNumberOrBoolean, }); -const metaDataSchemaObj = z.object({ - source: z.string(), - destination: z.string(), - fromRoot: z.boolean().default(false), - expand: z.optional( - z.object({ - regex: z.string(), - map: z.array(z.string()), - }) - ), -}); +export type RerouteFilterCondition = z.infer; -type MetadataSchema = z.infer; +export interface RerouteAndCondition { + and: RerouteCondition[]; +} -const metadataSchema = metaDataSchemaObj - .or( - z.string().transform( - (value) => - ({ - source: value, - destination: value, - fromRoot: false, - } as MetadataSchema) - ) - ) - .transform((metadata) => ({ - ...metadata, - destination: metadata.destination ?? metadata.source, - })) - .superRefine((value, ctx) => { - if (value.source.length === 0) { - ctx.addIssue({ - path: ['source'], - code: z.ZodIssueCode.custom, - message: 'source should not be empty', - }); - } - if (value.destination.length === 0) { - ctx.addIssue({ - path: ['destination'], - code: z.ZodIssueCode.custom, - message: 'destination should not be empty', - }); - } - }); +export interface RerouteOrCondition { + or: RerouteCondition[]; +} -export const apiScraperDefinitionSchema = z.object({ - id: z.string().regex(/^[\w-]+$/), - name: z.string(), - identityFields: z.array(z.string()), - metadata: z.array(metadataSchema), - metrics: z.array(apiMetricSchema), - source: z.object({ - type: z.literal('elasticsearch_api'), - endpoint: z.string(), - method: z.enum(['GET', 'POST']), - params: z.object({ - body: z.record(z.string(), z.any()), - query: z.record(z.string(), z.any()), - }), - collect: z.object({ - path: z.string(), - keyed: z.boolean().default(false), - }), - }), - managed: z.boolean().default(false), - apiKeyId: z.optional(z.string()), -}); +export type RerouteCondition = RerouteFilterCondition | RerouteAndCondition | RerouteOrCondition; -export type ApiScraperDefinition = z.infer; +export const rerouteConditionSchema: z.ZodType = z.lazy(() => + z.union([ + rerouteFilterConditionSchema, + z.object({ and: z.array(rerouteConditionSchema) }), + z.object({ or: z.array(rerouteConditionSchema) }), + ]) +); /** - * Example of a "root" StreamEntity + * Example of a "root" stream * { - * "id": "logs-all", - * "type": "logs", - * "dataset": "all", + * "id": "logs", * } * - * Example of a forked StreamEntity + * Example of a forked stream * { - * "id": "logs-nginx", - * "type": "logs", - * "dataset": "nginx", - * "forked_from": "logs-all" + * "id": "logs.nginx", + * "condition": { field: 'log.logger, operator: 'eq', value": "nginx_proxy" } + * "forked_from": "logs" * } */ export const streamDefinitonSchema = z.object({ id: z.string(), - type: z.enum(['logs', 'metrics']), - dataset: z.string(), forked_from: z.optional(z.string()), - condition: z.optional(z.string()), + condition: z.optional(rerouteConditionSchema), + root: z.boolean().default(false), }); export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/jest.config.js b/x-pack/plugins/streams/jest.config.js index 2bf0ab535784d..43d4fd28da9b5 100644 --- a/x-pack/plugins/streams/jest.config.js +++ b/x-pack/plugins/streams/jest.config.js @@ -8,11 +8,8 @@ module.exports = { preset: '@kbn/test', rootDir: '../../..', - roots: ['/x-pack/plugins/logsai/stream_entities_manager'], - coverageDirectory: - '/target/kibana-coverage/jest/x-pack/plugins/logsai/stream_entities_manager', + roots: ['/x-pack/plugins/streams'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/streams', coverageReporters: ['text', 'html'], - collectCoverageFrom: [ - '/x-pack/plugins/logsai/stream_entities_manager/{common,public,server}/**/*.{js,ts,tsx}', - ], + collectCoverageFrom: ['/x-pack/plugins/streams/{common,public,server}/**/*.{js,ts,tsx}'], }; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts index a2026c2105f37..e640339f1e456 100644 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts @@ -12,10 +12,10 @@ import { upsertIngestPipeline, upsertTemplate, } from '../../templates/manage_index_templates'; -import { logsAllLayer } from './component_templates/logs_all_layer'; -import { logsAllDefaultPipeline } from './ingest_pipelines/logs_all_default_pipeline'; -import { logsAllIndexTemplate } from './index_templates/logs_all'; -import { logsAllJsonPipeline } from './ingest_pipelines/logs_all_json_pipeline'; +import { logsLayer } from './component_templates/logs_layer'; +import { logsDefaultPipeline } from './ingest_pipelines/logs_default_pipeline'; +import { logsIndexTemplate } from './index_templates/logs'; +import { logsJsonPipeline } from './ingest_pipelines/logs_json_pipeline'; interface BootstrapRootEntityParams { esClient: ElasticsearchClient; @@ -23,8 +23,8 @@ interface BootstrapRootEntityParams { } export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { - await upsertComponent({ esClient, logger, component: logsAllLayer }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsAllJsonPipeline }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsAllDefaultPipeline }); - await upsertTemplate({ esClient, logger, template: logsAllIndexTemplate }); + await upsertComponent({ esClient, logger, component: logsLayer }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsJsonPipeline }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsDefaultPipeline }); + await upsertTemplate({ esClient, logger, template: logsIndexTemplate }); } diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index bd8ee24be4dc1..b3a63bf5cc4c5 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -10,7 +10,7 @@ import { ASSET_VERSION } from '../../../../common/constants'; export function generateLayer(id: string): ClusterPutComponentTemplateRequest { return { - name: `${id}@layer`, + name: `${id}@stream.layer`, template: { settings: { index: { @@ -30,7 +30,7 @@ export function generateLayer(id: string): ClusterPutComponentTemplateRequest { version: ASSET_VERSION, _meta: { managed: true, - description: `Default settings for the ${id} StreamEntity`, + description: `Default settings for the ${id} stream`, }, }; } diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts similarity index 98% rename from x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts rename to x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts index 33e43bce29544..2c2053fa6c82d 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts @@ -8,8 +8,8 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllLayer: ClusterPutComponentTemplateRequest = { - name: 'logs-all@layer', +export const logsLayer: ClusterPutComponentTemplateRequest = { + name: 'logs@stream.layer', template: { settings: { index: { @@ -32,17 +32,6 @@ export const logsAllLayer: ClusterPutComponentTemplateRequest = { '@timestamp': { type: 'date', }, - 'data_stream.namespace': { - type: 'constant_keyword', - }, - 'data_stream.dataset': { - type: 'constant_keyword', - value: 'all', - }, - 'data_stream.type': { - type: 'constant_keyword', - value: 'logs', - }, // Base labels: { @@ -1180,7 +1169,7 @@ export const logsAllLayer: ClusterPutComponentTemplateRequest = { version: ASSET_VERSION, _meta: { managed: true, - description: 'Default layer for logs-all StreamEntity', + description: 'Default layer for logs stream', }, deprecated: false, }; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts index b2e23e03dd7ef..a24c7357379fa 100644 --- a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts +++ b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts @@ -5,14 +5,9 @@ * 2.0. */ -import { ApiScraperDefinition } from '../../../../common/types'; - export class IdConflict extends Error { - public definition: ApiScraperDefinition; - - constructor(message: string, def: ApiScraperDefinition) { + constructor(message: string) { super(message); this.name = 'IdConflict'; - this.definition = def; } } diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts new file mode 100644 index 0000000000000..2f988204c74b0 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedStreamId extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedStreamId'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts new file mode 100644 index 0000000000000..243ea404ba66d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rerouteConditionToPainless } from './reroute_condition_to_painless'; + +const operatorConditionAndResutls = [ + { + condition: { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger == "nginx_proxy"', + }, + { + condition: { field: 'log.logger', operator: 'neq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger != "nginx_proxy"', + }, + { + condition: { field: 'http.response.status_code', operator: 'lt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code < 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'lte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code <= 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code > 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code >= 500', + }, + { + condition: { field: 'log.logger', operator: 'startsWith' as const, value: 'nginx' }, + result: 'ctx.log?.logger.startsWith("nginx")', + }, + { + condition: { field: 'log.logger', operator: 'endsWith' as const, value: 'proxy' }, + result: 'ctx.log?.logger.endsWith("proxy")', + }, + { + condition: { field: 'log.logger', operator: 'contains' as const, value: 'proxy' }, + result: 'ctx.log?.logger.contains("proxy")', + }, +]; + +describe('rerouteConditionToPainless', () => { + describe('operators', () => { + operatorConditionAndResutls.forEach((setup) => { + test(`${setup.condition.operator}`, () => { + expect(rerouteConditionToPainless(setup.condition)).toEqual(setup.result); + }); + }); + }); + + describe('and', () => { + test('simple', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('or', () => { + test('simple', () => { + const condition = { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" || ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('nested', () => { + test('and with a filter and or with 2 filters', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + test('and with 2 or with filters', () => { + const condition = { + and: [ + { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'service.name', operator: 'eq' as const, value: 'nginx' }, + ], + }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + '(ctx.log?.logger == "nginx_proxy" || ctx.service?.name == "nginx") && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + }); +}); diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts new file mode 100644 index 0000000000000..5df1e30097b58 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isBoolean, isString } from 'lodash'; +import { + RerouteAndCondition, + RerouteCondition, + rerouteConditionSchema, + RerouteFilterCondition, + rerouteFilterConditionSchema, + RerouteOrCondition, +} from '../../../../common/types'; + +function isFilterCondition(subject: any): subject is RerouteFilterCondition { + const result = rerouteFilterConditionSchema.safeParse(subject); + return result.success; +} + +function isAndCondition(subject: any): subject is RerouteAndCondition { + const result = rerouteConditionSchema.safeParse(subject); + return result.success && subject.and != null; +} + +function isOrCondition(subject: any): subject is RerouteOrCondition { + const result = rerouteConditionSchema.safeParse(subject); + return result.success && subject.or != null; +} + +function safePainlessField(condition: RerouteFilterCondition) { + return `ctx.${condition.field.split('.').join('?.')}`; +} + +function encodeValue(value: string | number | boolean) { + if (isString(value)) { + return `"${value}"`; + } + if (isBoolean(value)) { + return value ? 'true' : 'false'; + } + return value; +} + +function toPainless(condition: RerouteFilterCondition) { + switch (condition.operator) { + case 'neq': + return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; + case 'lt': + return `${safePainlessField(condition)} < ${encodeValue(condition.value)}`; + case 'lte': + return `${safePainlessField(condition)} <= ${encodeValue(condition.value)}`; + case 'gt': + return `${safePainlessField(condition)} > ${encodeValue(condition.value)}`; + case 'gte': + return `${safePainlessField(condition)} >= ${encodeValue(condition.value)}`; + case 'startsWith': + return `${safePainlessField(condition)}.startsWith(${encodeValue(condition.value)})`; + case 'endsWith': + return `${safePainlessField(condition)}.endsWith(${encodeValue(condition.value)})`; + case 'contains': + return `${safePainlessField(condition)}.contains(${encodeValue(condition.value)})`; + default: + return `${safePainlessField(condition)} == ${encodeValue(condition.value)}`; + } +} + +export function rerouteConditionToPainless(condition: RerouteCondition, nested = false): string { + if (isFilterCondition(condition)) { + return toPainless(condition); + } + if (isAndCondition(condition)) { + const and = condition.and + .map((filter) => rerouteConditionToPainless(filter, true)) + .join(' && '); + return nested ? `(${and})` : and; + } + if (isOrCondition(condition)) { + const or = condition.or.map((filter) => rerouteConditionToPainless(filter, true)).join(' || '); + return nested ? `(${or})` : or; + } + return ''; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 5052aacf99b6b..14972ce8a6380 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -13,14 +13,14 @@ export function generateIndexTemplate( ignoreMissing: string[] = [] ) { return { - name: id, - index_patterns: [`${id}-*`], - composed_of: [...composedOf, `${id}@layer`], + name: `${id}@stream`, + index_patterns: [id], + composed_of: [...composedOf, `${id}@stream.layer`], priority: 200, version: ASSET_VERSION, _meta: { managed: true, - description: `The index template for ${id} StreamEntity`, + description: `The index template for ${id} stream`, }, data_stream: { hidden: false, @@ -28,11 +28,11 @@ export function generateIndexTemplate( template: { settings: { index: { - default_pipeline: `${id}@default-pipeline`, + default_pipeline: `${id}@stream.default-pipeline`, }, }, }, allow_auto_create: true, - ignore_missing_component_templates: [...ignoreMissing, `${id}@layer`], + ignore_missing_component_templates: [...ignoreMissing, `${id}@stream.layer`], }; } diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts similarity index 79% rename from x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts rename to x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts index b04b9b8e6841d..58324213dae16 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts @@ -8,5 +8,4 @@ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { generateIndexTemplate } from './generate_index_template'; -export const logsAllIndexTemplate: IndicesPutIndexTemplateRequest = - generateIndexTemplate('logs-all'); +export const logsIndexTemplate: IndicesPutIndexTemplateRequest = generateIndexTemplate('logs'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 5411579c285bc..1cd7067014a95 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -9,23 +9,17 @@ import { ASSET_VERSION } from '../../../../common/constants'; export function generateIngestPipeline(id: string) { return { - id: `${id}@default-pipeline`, + id: `${id}@stream.default-pipeline`, processors: [ - { - append: { - field: 'labels.elastic.stream_entities', - value: [id], - }, - }, { pipeline: { - name: `${id}@reroutes`, + name: `${id}@stream.reroutes`, ignore_missing_pipeline: true, }, }, ], _meta: { - description: `Default pipeline for the ${id} StreamEntity`, + description: `Default pipeline for the ${id} streams`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index a8f0a84ca27d2..43dc8cb787f5a 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; +import { rerouteConditionToPainless } from '../helpers/reroute_condition_to_painless'; interface GenerateReroutePipelineParams { esClient: ElasticsearchClient; @@ -24,17 +25,25 @@ export async function generateReroutePipeline({ }); return { - id: `${definition.id}@reroutes`, + id: `${definition.id}@stream.reroutes`, processors: response.hits.hits.map((doc) => { + if (!doc._source) { + throw new Error('Source missing for stream definiton document'); + } + if (!doc._source.condition) { + throw new Error( + `Reroute condition missing from forked stream definition ${doc._source.id}` + ); + } return { reroute: { - dataset: doc._source.dataset, - if: doc._source.condition, + destination: doc._source.id, + if: rerouteConditionToPainless(doc._source.condition), }, }; }), _meta: { - description: `Reoute pipeline for the ${definition.id} StreamEntity`, + description: `Reoute pipeline for the ${definition.id} stream`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts similarity index 72% rename from x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts rename to x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts index 333e752ec9762..9d9edf3e92b4a 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts @@ -7,8 +7,8 @@ import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllDefaultPipeline = { - id: 'logs-all@default-pipeline', +export const logsDefaultPipeline = { + id: 'logs@stream.default-pipeline', processors: [ { set: { @@ -24,27 +24,21 @@ export const logsAllDefaultPipeline = { value: '{{{_ingest.timestamp}}}', }, }, - { - append: { - field: 'labels.elastic.stream_entities', - value: ['logs-all'], - }, - }, { pipeline: { - name: 'logs-all@json-pipeline', + name: 'logs@stream.json-pipeline', ignore_missing_pipeline: true, }, }, { pipeline: { - name: 'logs-all@reroutes', + name: 'logs@stream.reroutes', ignore_missing_pipeline: true, }, }, ], _meta: { - description: 'Default pipeline for the logs-all StreamEntity', + description: 'Default pipeline for the logs stream', managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts similarity index 95% rename from x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts rename to x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts index 276b981aafbfd..2b6fb7d8d9344 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts @@ -7,8 +7,8 @@ import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllJsonPipeline = { - id: 'logs-all@json-pipeline', +export const logsJsonPipeline = { + id: 'logs@stream.json-pipeline', processors: [ { rename: { diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 1a10a9d5ee807..4d95632df9f9e 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -8,7 +8,6 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { - id: 'logs-all', - type: 'logs', - dataset: 'all', + id: 'logs', + root: true, }; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 0294524815d21..47e2b4061c063 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -33,23 +33,27 @@ interface ReadStreamParams extends BaseParams { } export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { - const response = await scopedClusterClient.asCurrentUser.get({ - id, - index: STREAMS_INDEX, - }); - if (!response.found) { - throw new DefinitionNotFound(`Stream Entity definition for ${id} not found.`); + try { + const response = await scopedClusterClient.asCurrentUser.get({ + id, + index: STREAMS_INDEX, + }); + const definition = response._source as StreamDefinition; + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); + const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); + return { + definition, + index_template: indexTemplate, + component_template: componentTemplate, + ingest_pipelines: ingestPipelines, + }; + } catch (e) { + if (e.meta?.statusCode === 404) { + throw new DefinitionNotFound(`Stream definition for ${id} not found.`); + } + throw e; } - const definition = response._source as StreamDefinition; - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); - const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); - return { - definition, - index_template: indexTemplate, - component_template: componentTemplate, - ingest_pipelines: ingestPipelines, - }; } export async function readIndexTemplate({ @@ -57,9 +61,11 @@ export async function readIndexTemplate({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ - name: definition.id, + name: `${definition.id}@stream`, }); - const indexTemplate = response.index_templates.find((doc) => doc.name === definition.id); + const indexTemplate = response.index_templates.find( + (doc) => doc.name === `${definition.id}@stream` + ); if (!indexTemplate) { throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); } @@ -71,10 +77,10 @@ export async function readComponentTemplate({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ - name: `${definition.id}@layer`, + name: `${definition.id}@stream.layer`, }); const componentTemplate = response.component_templates.find( - (doc) => doc.name === `${definition.id}@layer` + (doc) => doc.name === `${definition.id}@stream.layer` ); if (!componentTemplate) { throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); @@ -87,7 +93,7 @@ export async function readIngestPipelines({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ - id: `${definition.id}*`, + id: `${definition.id}@stream.*`, }); return response; diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1d21b4194f172..198c2e88c4c52 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -16,6 +16,7 @@ import { createServerRoute } from '../create_server_route'; import { streamDefinitonSchema } from '../../../common/types'; import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; import { createStream, readStream } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', @@ -46,6 +47,12 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); + if (!params.body.id.startsWith(rootDefinition.id)) { + throw new MalformedStreamId( + `The ID (${params.body.id}) from the new stream must start with the parent's id (${rootDefinition.id})` + ); + } + await createStream({ scopedClusterClient, definition: { ...params.body, forked_from: rootDefinition.id }, @@ -64,7 +71,11 @@ export const forkStreamsRoute = createServerRoute({ return response.notFound({ body: e }); } - if (e instanceof SecurityException || e instanceof ForkConditionMissing) { + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { return response.customError({ body: e, statusCode: 400 }); } diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index c2f7cfe59b3ec..061522787de52 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -36,6 +36,8 @@ export const readStreamRoute = createServerRoute({ if (e instanceof DefinitionNotFound) { return response.notFound({ body: e }); } + + return response.customError({ body: e, statusCode: 500 }); } }, }); diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts index a4744c6962155..fac220fb8be1f 100644 --- a/x-pack/plugins/streams/server/templates/components/base.ts +++ b/x-pack/plugins/streams/server/templates/components/base.ts @@ -37,13 +37,15 @@ export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { ignore_above: 1024, type: 'keyword', }, + root: { + type: 'boolean', + }, forked_from: { ignore_above: 1024, type: 'keyword', }, condition: { - ignore_above: 1024, - type: 'keyword', + type: 'object', }, event: { properties: { From 9b338087e0852d8c329164cbff1a5aae9af2e9ac Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 1 Nov 2024 12:00:54 -0600 Subject: [PATCH 03/18] ensure forked streams are marked root:false --- x-pack/plugins/streams/server/routes/streams/fork.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 198c2e88c4c52..c019abd17b58d 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -55,7 +55,7 @@ export const forkStreamsRoute = createServerRoute({ await createStream({ scopedClusterClient, - definition: { ...params.body, forked_from: rootDefinition.id }, + definition: { ...params.body, forked_from: rootDefinition.id, root: false }, }); await bootstrapStream({ From d99a3ff546e6bcd716faad58a1f770ea859753e1 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 17:41:32 +0100 Subject: [PATCH 04/18] stream management --- x-pack/plugins/streams/common/types.ts | 63 +- .../lib/streams/bootstrap_root_assets.ts | 30 - .../server/lib/streams/bootstrap_stream.ts | 62 - .../component_templates/generate_layer.ts | 35 +- .../streams/component_templates/logs_layer.ts | 1172 +---------------- ....test.ts => condition_to_painless.test.ts} | 14 +- ...o_painless.ts => condition_to_painless.ts} | 34 +- .../server/lib/streams/helpers/hierarchy.ts | 31 + .../lib/streams/index_templates/logs.ts | 11 - .../generate_ingest_pipeline.ts | 18 +- .../generate_reroute_pipeline.ts | 30 +- .../ingest_pipelines/logs_default_pipeline.ts | 48 +- .../ingest_pipelines/logs_json_pipeline.ts | 58 - .../lib/streams/root_stream_definition.ts | 20 + .../streams/server/lib/streams/stream_crud.ts | 121 +- x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/edit.ts | 162 +++ .../streams/server/routes/streams/enable.ts | 10 +- .../streams/server/routes/streams/fork.ts | 36 +- 19 files changed, 468 insertions(+), 1489 deletions(-) delete mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts rename x-pack/plugins/streams/server/lib/streams/helpers/{reroute_condition_to_painless.test.ts => condition_to_painless.test.ts} (89%) rename x-pack/plugins/streams/server/lib/streams/helpers/{reroute_condition_to_painless.ts => condition_to_painless.ts} (68%) create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/edit.ts diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 5bdc018452376..0019eb8fee054 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -9,32 +9,62 @@ import { z } from '@kbn/zod'; const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); -export const rerouteFilterConditionSchema = z.object({ +export const filterConditionSchema = z.object({ field: z.string(), operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), value: stringOrNumberOrBoolean, }); -export type RerouteFilterCondition = z.infer; +export type FilterCondition = z.infer; -export interface RerouteAndCondition { - and: RerouteCondition[]; +export interface AndCondition { + and: Condition[]; } export interface RerouteOrCondition { - or: RerouteCondition[]; + or: Condition[]; } -export type RerouteCondition = RerouteFilterCondition | RerouteAndCondition | RerouteOrCondition; +export type Condition = FilterCondition | AndCondition | RerouteOrCondition | undefined; -export const rerouteConditionSchema: z.ZodType = z.lazy(() => +export const conditionSchema: z.ZodType = z.lazy(() => z.union([ - rerouteFilterConditionSchema, - z.object({ and: z.array(rerouteConditionSchema) }), - z.object({ or: z.array(rerouteConditionSchema) }), + filterConditionSchema, + z.object({ and: z.array(conditionSchema) }), + z.object({ or: z.array(conditionSchema) }), ]) ); +export const grokProcessingDefinitionSchema = z.object({ + type: z.literal('grok'), + field: z.string(), + patterns: z.array(z.string()), + pattern_definitions: z.optional(z.record(z.string())), +}); + +export const dissectProcessingDefinitionSchema = z.object({ + type: z.literal('dissect'), + field: z.string(), + pattern: z.string(), +}); + +export const processingDefinitionSchema = z.object({ + condition: z.optional(conditionSchema), + config: z.discriminatedUnion('type', [ + grokProcessingDefinitionSchema, + dissectProcessingDefinitionSchema, + ]), +}); + +export type ProcessingDefinition = z.infer; + +export const fieldDefinitionSchema = z.object({ + name: z.string(), + type: z.enum(['keyword', 'text', 'long', 'double', 'date', 'boolean', 'ip']), +}); + +export type FieldDefinition = z.infer; + /** * Example of a "root" stream * { @@ -51,9 +81,16 @@ export const rerouteConditionSchema: z.ZodType = z.lazy(() => export const streamDefinitonSchema = z.object({ id: z.string(), - forked_from: z.optional(z.string()), - condition: z.optional(rerouteConditionSchema), - root: z.boolean().default(false), + processing: z.array(processingDefinitionSchema).default([]), + fields: z.array(fieldDefinitionSchema).default([]), + children: z + .array( + z.object({ + id: z.string(), + condition: conditionSchema, + }) + ) + .default([]), }); export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts deleted file mode 100644 index e640339f1e456..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Logger } from '@kbn/logging'; -import { - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; -import { logsLayer } from './component_templates/logs_layer'; -import { logsDefaultPipeline } from './ingest_pipelines/logs_default_pipeline'; -import { logsIndexTemplate } from './index_templates/logs'; -import { logsJsonPipeline } from './ingest_pipelines/logs_json_pipeline'; - -interface BootstrapRootEntityParams { - esClient: ElasticsearchClient; - logger: Logger; -} - -export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { - await upsertComponent({ esClient, logger, component: logsLayer }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsJsonPipeline }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsDefaultPipeline }); - await upsertTemplate({ esClient, logger, template: logsIndexTemplate }); -} diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts deleted file mode 100644 index c609af5b49978..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; -import { Logger } from '@kbn/logging'; -import { StreamDefinition } from '../../../common/types'; -import { generateLayer } from './component_templates/generate_layer'; -import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; -import { - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; -import { generateIndexTemplate } from './index_templates/generate_index_template'; -import { getIndexTemplateComponents } from './stream_crud'; -import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; - -interface BootstrapStreamParams { - scopedClusterClient: IScopedClusterClient; - definition: StreamDefinition; - rootDefinition: StreamDefinition; - logger: Logger; -} -export async function bootstrapStream({ - scopedClusterClient, - definition, - rootDefinition, - logger, -}: BootstrapStreamParams) { - const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ - scopedClusterClient, - definition: rootDefinition, - }); - const reroutePipeline = await generateReroutePipeline({ - esClient: scopedClusterClient.asCurrentUser, - definition: rootDefinition, - }); - await upsertComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - component: generateLayer(definition.id), - }); - await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - pipeline: generateIngestPipeline(definition.id), - }); - await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - pipeline: reroutePipeline, - }); - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), - }); -} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index b3a63bf5cc4c5..f7398cb1304a8 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -5,26 +5,31 @@ * 2.0. */ -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + ClusterPutComponentTemplateRequest, + MappingProperty, +} from '@elastic/elasticsearch/lib/api/types'; +import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; +import { logsSettings } from './logs_layer'; +import { isRoot } from '../helpers/hierarchy'; -export function generateLayer(id: string): ClusterPutComponentTemplateRequest { +export function generateLayer( + id: string, + definition: StreamDefinition +): ClusterPutComponentTemplateRequest { + const properties: Record = {}; + definition.fields.forEach((field) => { + properties[field.name] = { + type: field.type, + }; + }); return { name: `${id}@stream.layer`, template: { - settings: { - index: { - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - mapping: { - total_fields: { - ignore_dynamic_beyond_limit: true, - }, - ignore_malformed: true, - }, - }, + settings: isRoot(definition.id) ? logsSettings : {}, + mappings: { + properties, }, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts index 2c2053fa6c82d..6b41d04131c56 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts @@ -5,1171 +5,19 @@ * 2.0. */ -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { ASSET_VERSION } from '../../../../common/constants'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; -export const logsLayer: ClusterPutComponentTemplateRequest = { - name: 'logs@stream.layer', - template: { - settings: { - index: { - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - mapping: { - total_fields: { - ignore_dynamic_beyond_limit: true, - }, - ignore_malformed: true, - }, - }, +export const logsSettings: IndicesIndexSettings = { + index: { + lifecycle: { + name: 'logs', }, - mappings: { - dynamic: false, - date_detection: false, - properties: { - '@timestamp': { - type: 'date', - }, - - // Base - labels: { - type: 'object', - }, - message: { - type: 'match_only_text', - }, - tags: { - ignore_above: 1024, - type: 'keyword', - }, - event: { - properties: { - ingested: { - type: 'date', - }, - }, - }, - - // file - file: { - properties: { - accessed: { - type: 'date', - }, - attributes: { - ignore_above: 1024, - type: 'keyword', - }, - code_signature: { - properties: { - digest_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - exists: { - type: 'boolean', - }, - signing_id: { - ignore_above: 1024, - type: 'keyword', - }, - status: { - ignore_above: 1024, - type: 'keyword', - }, - subject_name: { - ignore_above: 1024, - type: 'keyword', - }, - team_id: { - ignore_above: 1024, - type: 'keyword', - }, - timestamp: { - type: 'date', - }, - trusted: { - type: 'boolean', - }, - valid: { - type: 'boolean', - }, - }, - }, - created: { - type: 'date', - }, - ctime: { - type: 'date', - }, - device: { - ignore_above: 1024, - type: 'keyword', - }, - directory: { - ignore_above: 1024, - type: 'keyword', - }, - drive_letter: { - ignore_above: 1, - type: 'keyword', - }, - elf: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - byte_order: { - ignore_above: 1024, - type: 'keyword', - }, - cpu_type: { - ignore_above: 1024, - type: 'keyword', - }, - creation_date: { - type: 'date', - }, - exports: { - type: 'flattened', - }, - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - header: { - properties: { - abi_version: { - ignore_above: 1024, - type: 'keyword', - }, - class: { - ignore_above: 1024, - type: 'keyword', - }, - data: { - ignore_above: 1024, - type: 'keyword', - }, - entrypoint: { - type: 'long', - }, - object_version: { - ignore_above: 1024, - type: 'keyword', - }, - os_abi: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - sections: { - properties: { - chi2: { - type: 'long', - }, - entropy: { - type: 'long', - }, - flags: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_offset: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - var_entropy: { - type: 'long', - }, - virtual_address: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - segments: { - properties: { - sections: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - type: 'nested', - }, - shared_libraries: { - ignore_above: 1024, - type: 'keyword', - }, - telfhash: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - extension: { - ignore_above: 1024, - type: 'keyword', - }, - fork_name: { - ignore_above: 1024, - type: 'keyword', - }, - gid: { - ignore_above: 1024, - type: 'keyword', - }, - group: { - ignore_above: 1024, - type: 'keyword', - }, - hash: { - properties: { - md5: { - ignore_above: 1024, - type: 'keyword', - }, - sha1: { - ignore_above: 1024, - type: 'keyword', - }, - sha256: { - ignore_above: 1024, - type: 'keyword', - }, - sha384: { - ignore_above: 1024, - type: 'keyword', - }, - sha512: { - ignore_above: 1024, - type: 'keyword', - }, - ssdeep: { - ignore_above: 1024, - type: 'keyword', - }, - tlsh: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - inode: { - ignore_above: 1024, - type: 'keyword', - }, - macho: { - properties: { - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - sections: { - properties: { - entropy: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - var_entropy: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - symhash: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - mime_type: { - ignore_above: 1024, - type: 'keyword', - }, - mode: { - ignore_above: 1024, - type: 'keyword', - }, - mtime: { - type: 'date', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - owner: { - ignore_above: 1024, - type: 'keyword', - }, - path: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - pe: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - company: { - ignore_above: 1024, - type: 'keyword', - }, - description: { - ignore_above: 1024, - type: 'keyword', - }, - file_version: { - ignore_above: 1024, - type: 'keyword', - }, - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - imphash: { - ignore_above: 1024, - type: 'keyword', - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - original_file_name: { - ignore_above: 1024, - type: 'keyword', - }, - pehash: { - ignore_above: 1024, - type: 'keyword', - }, - product: { - ignore_above: 1024, - type: 'keyword', - }, - sections: { - properties: { - entropy: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - var_entropy: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - }, - }, - size: { - type: 'long', - }, - target_path: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - uid: { - ignore_above: 1024, - type: 'keyword', - }, - x509: { - properties: { - alternative_names: { - ignore_above: 1024, - type: 'keyword', - }, - issuer: { - properties: { - common_name: { - ignore_above: 1024, - type: 'keyword', - }, - country: { - ignore_above: 1024, - type: 'keyword', - }, - distinguished_name: { - ignore_above: 1024, - type: 'keyword', - }, - locality: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - organizational_unit: { - ignore_above: 1024, - type: 'keyword', - }, - state_or_province: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - not_after: { - type: 'date', - }, - not_before: { - type: 'date', - }, - public_key_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - public_key_curve: { - ignore_above: 1024, - type: 'keyword', - }, - public_key_exponent: { - doc_values: false, - index: false, - type: 'long', - }, - public_key_size: { - type: 'long', - }, - serial_number: { - ignore_above: 1024, - type: 'keyword', - }, - signature_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - subject: { - properties: { - common_name: { - ignore_above: 1024, - type: 'keyword', - }, - country: { - ignore_above: 1024, - type: 'keyword', - }, - distinguished_name: { - ignore_above: 1024, - type: 'keyword', - }, - locality: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - organizational_unit: { - ignore_above: 1024, - type: 'keyword', - }, - state_or_province: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - version_number: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - - // Host - host: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - cpu: { - properties: { - usage: { - scaling_factor: 1000, - type: 'scaled_float', - }, - }, - }, - disk: { - properties: { - read: { - properties: { - bytes: { - type: 'long', - }, - }, - }, - write: { - properties: { - bytes: { - type: 'long', - }, - }, - }, - }, - }, - domain: { - ignore_above: 1024, - type: 'keyword', - }, - geo: { - properties: { - city_name: { - ignore_above: 1024, - type: 'keyword', - }, - continent_code: { - ignore_above: 1024, - type: 'keyword', - }, - continent_name: { - ignore_above: 1024, - type: 'keyword', - }, - country_iso_code: { - ignore_above: 1024, - type: 'keyword', - }, - country_name: { - ignore_above: 1024, - type: 'keyword', - }, - location: { - type: 'geo_point', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - postal_code: { - ignore_above: 1024, - type: 'keyword', - }, - region_iso_code: { - ignore_above: 1024, - type: 'keyword', - }, - region_name: { - ignore_above: 1024, - type: 'keyword', - }, - timezone: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - hostname: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - ip: { - type: 'ip', - }, - mac: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - network: { - properties: { - egress: { - properties: { - bytes: { - type: 'long', - }, - packets: { - type: 'long', - }, - }, - }, - ingress: { - properties: { - bytes: { - type: 'long', - }, - packets: { - type: 'long', - }, - }, - }, - }, - }, - os: { - properties: { - family: { - ignore_above: 1024, - type: 'keyword', - }, - full: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - kernel: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - platform: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - uptime: { - type: 'long', - }, - }, - }, - - // Orchestrator - orchestrator: { - properties: { - api_version: { - ignore_above: 1024, - type: 'keyword', - }, - cluster: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - url: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - namespace: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - resource: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - log: { - properties: { - file: { - properties: { - path: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - level: { - ignore_above: 1024, - type: 'keyword', - }, - logger: { - ignore_above: 1024, - type: 'keyword', - }, - origin: { - properties: { - file: { - properties: { - line: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - function: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - syslog: { - properties: { - facility: { - properties: { - code: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - priority: { - type: 'long', - }, - severity: { - properties: { - code: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - type: 'object', - }, - }, - }, - - // ECS - ecs: { - properties: { - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - // Agent - agent: { - properties: { - build: { - properties: { - original: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - ephemeral_id: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - // Cloud - cloud: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - origin: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - target: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - }, - }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, }, + ignore_malformed: true, }, }, - version: ASSET_VERSION, - _meta: { - managed: true, - description: 'Default layer for logs stream', - }, - deprecated: false, }; diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts similarity index 89% rename from x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts rename to x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts index 243ea404ba66d..aab7f27f12d14 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { rerouteConditionToPainless } from './reroute_condition_to_painless'; +import { conditionToPainless } from './condition_to_painless'; const operatorConditionAndResutls = [ { @@ -46,11 +46,11 @@ const operatorConditionAndResutls = [ }, ]; -describe('rerouteConditionToPainless', () => { +describe('conditionToPainless', () => { describe('operators', () => { operatorConditionAndResutls.forEach((setup) => { test(`${setup.condition.operator}`, () => { - expect(rerouteConditionToPainless(setup.condition)).toEqual(setup.result); + expect(conditionToPainless(setup.condition)).toEqual(setup.result); }); }); }); @@ -64,7 +64,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" && ctx.log?.level == "error"' ) ); @@ -80,7 +80,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" || ctx.log?.level == "error"' ) ); @@ -101,7 +101,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' ) ); @@ -124,7 +124,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( '(ctx.log?.logger == "nginx_proxy" || ctx.service?.name == "nginx") && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' ) ); diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts similarity index 68% rename from x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts rename to x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts index 5df1e30097b58..539ad3603535b 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts @@ -7,30 +7,30 @@ import { isBoolean, isString } from 'lodash'; import { - RerouteAndCondition, - RerouteCondition, - rerouteConditionSchema, - RerouteFilterCondition, - rerouteFilterConditionSchema, + AndCondition, + Condition, + conditionSchema, + FilterCondition, + filterConditionSchema, RerouteOrCondition, } from '../../../../common/types'; -function isFilterCondition(subject: any): subject is RerouteFilterCondition { - const result = rerouteFilterConditionSchema.safeParse(subject); +function isFilterCondition(subject: any): subject is FilterCondition { + const result = filterConditionSchema.safeParse(subject); return result.success; } -function isAndCondition(subject: any): subject is RerouteAndCondition { - const result = rerouteConditionSchema.safeParse(subject); +function isAndCondition(subject: any): subject is AndCondition { + const result = conditionSchema.safeParse(subject); return result.success && subject.and != null; } function isOrCondition(subject: any): subject is RerouteOrCondition { - const result = rerouteConditionSchema.safeParse(subject); + const result = conditionSchema.safeParse(subject); return result.success && subject.or != null; } -function safePainlessField(condition: RerouteFilterCondition) { +function safePainlessField(condition: FilterCondition) { return `ctx.${condition.field.split('.').join('?.')}`; } @@ -44,7 +44,7 @@ function encodeValue(value: string | number | boolean) { return value; } -function toPainless(condition: RerouteFilterCondition) { +function toPainless(condition: FilterCondition) { switch (condition.operator) { case 'neq': return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; @@ -67,19 +67,17 @@ function toPainless(condition: RerouteFilterCondition) { } } -export function rerouteConditionToPainless(condition: RerouteCondition, nested = false): string { +export function conditionToPainless(condition: Condition, nested = false): string { if (isFilterCondition(condition)) { return toPainless(condition); } if (isAndCondition(condition)) { - const and = condition.and - .map((filter) => rerouteConditionToPainless(filter, true)) - .join(' && '); + const and = condition.and.map((filter) => conditionToPainless(filter, true)).join(' && '); return nested ? `(${and})` : and; } if (isOrCondition(condition)) { - const or = condition.or.map((filter) => rerouteConditionToPainless(filter, true)).join(' || '); + const or = condition.or.map((filter) => conditionToPainless(filter, true)).join(' || '); return nested ? `(${or})` : or; } - return ''; + return 'false'; } diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts new file mode 100644 index 0000000000000..eeb69f89473a1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../../common/types'; + +export function isDescendandOf(parent: StreamDefinition, child: StreamDefinition) { + return child.id.startsWith(parent.id); +} + +// streams are named by dots: logs.a.b.c - a child means there is only one level of dots added +export function isChildOf(parent: StreamDefinition, child: StreamDefinition) { + return ( + isDescendandOf(parent, child) && child.id.split('.').length === parent.id.split('.').length + 1 + ); +} + +export function getParentId(id: string) { + const parts = id.split('.'); + if (parts.length === 1) { + return undefined; + } + return parts.slice(0, parts.length - 1).join('.'); +} + +export function isRoot(id: string) { + return id.split('.').length === 1; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts deleted file mode 100644 index 58324213dae16..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { generateIndexTemplate } from './generate_index_template'; - -export const logsIndexTemplate: IndicesPutIndexTemplateRequest = generateIndexTemplate('logs'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 1cd7067014a95..146671e79744f 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -5,12 +5,26 @@ * 2.0. */ +import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; +import { logsDefaultPipelineProcessors } from './logs_default_pipeline'; +import { isRoot } from '../helpers/hierarchy'; -export function generateIngestPipeline(id: string) { +export function generateIngestPipeline(id: string, definition: StreamDefinition) { return { id: `${id}@stream.default-pipeline`, processors: [ + ...(isRoot(definition.id) ? logsDefaultPipelineProcessors : []), + ...definition.processing.map((processor) => { + const { type, ...config } = processor.config; + return { + [type]: { + ...config, + if: processor.condition ? conditionToPainless(processor.condition) : undefined, + }, + }; + }), { pipeline: { name: `${id}@stream.reroutes`, @@ -19,7 +33,7 @@ export function generateIngestPipeline(id: string) { }, ], _meta: { - description: `Default pipeline for the ${id} streams`, + description: `Default pipeline for the ${id} stream`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index 43dc8cb787f5a..b73dc61b148ed 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -5,40 +5,22 @@ * 2.0. */ -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { StreamDefinition } from '../../../../common/types'; -import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; -import { rerouteConditionToPainless } from '../helpers/reroute_condition_to_painless'; +import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; interface GenerateReroutePipelineParams { - esClient: ElasticsearchClient; definition: StreamDefinition; } -export async function generateReroutePipeline({ - esClient, - definition, -}: GenerateReroutePipelineParams) { - const response = await esClient.search({ - index: STREAMS_INDEX, - query: { match: { forked_from: definition.id } }, - }); - +export async function generateReroutePipeline({ definition }: GenerateReroutePipelineParams) { return { id: `${definition.id}@stream.reroutes`, - processors: response.hits.hits.map((doc) => { - if (!doc._source) { - throw new Error('Source missing for stream definiton document'); - } - if (!doc._source.condition) { - throw new Error( - `Reroute condition missing from forked stream definition ${doc._source.id}` - ); - } + processors: definition.children.map((child) => { return { reroute: { - destination: doc._source.id, - if: rerouteConditionToPainless(doc._source.condition), + destination: child.id, + if: conditionToPainless(child.condition), }, }; }), diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts index 9d9edf3e92b4a..762155ba5047c 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts @@ -5,41 +5,19 @@ * 2.0. */ -import { ASSET_VERSION } from '../../../../common/constants'; - -export const logsDefaultPipeline = { - id: 'logs@stream.default-pipeline', - processors: [ - { - set: { - description: "If '@timestamp' is missing, set it with the ingest timestamp", - field: '@timestamp', - override: false, - copy_from: '_ingest.timestamp', - }, - }, - { - set: { - field: 'event.ingested', - value: '{{{_ingest.timestamp}}}', - }, +export const logsDefaultPipelineProcessors = [ + { + set: { + description: "If '@timestamp' is missing, set it with the ingest timestamp", + field: '@timestamp', + override: false, + copy_from: '_ingest.timestamp', }, - { - pipeline: { - name: 'logs@stream.json-pipeline', - ignore_missing_pipeline: true, - }, - }, - { - pipeline: { - name: 'logs@stream.reroutes', - ignore_missing_pipeline: true, - }, + }, + { + pipeline: { + name: 'logs@json-pipeline', + ignore_missing_pipeline: true, }, - ], - _meta: { - description: 'Default pipeline for the logs stream', - managed: true, }, - version: ASSET_VERSION, -}; +]; diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts deleted file mode 100644 index 2b6fb7d8d9344..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ASSET_VERSION } from '../../../../common/constants'; - -export const logsJsonPipeline = { - id: 'logs@stream.json-pipeline', - processors: [ - { - rename: { - if: "ctx.message instanceof String && ctx.message.startsWith('{') && ctx.message.endsWith('}')", - field: 'message', - target_field: '_tmp_json_message', - ignore_missing: true, - }, - }, - { - json: { - if: 'ctx._tmp_json_message != null', - field: '_tmp_json_message', - add_to_root: true, - add_to_root_conflict_strategy: 'merge' as const, - allow_duplicate_keys: true, - on_failure: [ - { - rename: { - field: '_tmp_json_message', - target_field: 'message', - ignore_missing: true, - }, - }, - ], - }, - }, - { - dot_expander: { - if: 'ctx._tmp_json_message != null', - field: '*', - override: true, - }, - }, - { - remove: { - field: '_tmp_json_message', - ignore_missing: true, - }, - }, - ], - _meta: { - description: 'automatic parsing of JSON log messages', - managed: true, - }, - version: ASSET_VERSION, -}; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 4d95632df9f9e..3555146ccc008 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -10,4 +10,24 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { id: 'logs', root: true, + processing: [], + children: [], + fields: [ + { + name: '@timestamp', + type: 'date', + }, + { + name: 'message', + type: 'text', + }, + { + name: 'host.name', + type: 'keyword', + }, + { + name: 'log.level', + type: 'keyword', + }, + ], }; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 47e2b4061c063..5fe9e5f6de0df 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -7,9 +7,19 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { get } from 'lodash'; +import { Logger } from '@kbn/logging'; import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; -import { ComponentTemplateNotFound, DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { generateLayer } from './component_templates/generate_layer'; +import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; +import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; +import { generateIndexTemplate } from './index_templates/generate_index_template'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -19,7 +29,7 @@ interface BaseParamsWithDefinition extends BaseParams { definition: StreamDefinition; } -export async function createStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { +async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, index: STREAMS_INDEX, @@ -39,14 +49,8 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) index: STREAMS_INDEX, }); const definition = response._source as StreamDefinition; - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); - const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); return { definition, - index_template: indexTemplate, - component_template: componentTemplate, - ingest_pipelines: ingestPipelines, }; } catch (e) { if (e.meta?.statusCode === 404) { @@ -56,6 +60,18 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) } } +export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamParams) { + try { + await readStream({ id, scopedClusterClient }); + return true; + } catch (e) { + if (e instanceof DefinitionNotFound) { + return false; + } + throw e; + } +} + export async function readIndexTemplate({ scopedClusterClient, definition, @@ -72,33 +88,6 @@ export async function readIndexTemplate({ return indexTemplate; } -export async function readComponentTemplate({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ - name: `${definition.id}@stream.layer`, - }); - const componentTemplate = response.component_templates.find( - (doc) => doc.name === `${definition.id}@stream.layer` - ); - if (!componentTemplate) { - throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); - } - return componentTemplate; -} - -export async function readIngestPipelines({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ - id: `${definition.id}@stream.*`, - }); - - return response; -} - export async function getIndexTemplateComponents({ scopedClusterClient, definition, @@ -113,3 +102,65 @@ export async function getIndexTemplateComponents({ ) as string[], }; } + +interface SyncStreamParams { + scopedClusterClient: IScopedClusterClient; + definition: StreamDefinition; + rootDefinition?: StreamDefinition; + logger: Logger; +} + +export async function syncStream({ + scopedClusterClient, + definition, + rootDefinition, + logger, +}: SyncStreamParams) { + await upsertStream({ + scopedClusterClient, + definition, + }); + await upsertComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + component: generateLayer(definition.id, definition), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: generateIngestPipeline(definition.id, definition), + }); + const reroutePipeline = await generateReroutePipeline({ + definition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: reroutePipeline, + }); + if (rootDefinition) { + const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ + scopedClusterClient, + definition: rootDefinition, + }); + const parentReroutePipeline = await generateReroutePipeline({ + definition: rootDefinition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: parentReroutePipeline, + }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), + }); + } else { + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id), + }); + } +} diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index c24362f3ec8a9..544b60ccd404c 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { editStreamRoute } from './streams/edit'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; import { readStreamRoute } from './streams/read'; @@ -13,6 +14,7 @@ export const StreamsRouteRepository = { ...enableStreamsRoute, ...forkStreamsRoute, ...readStreamRoute, + ...editStreamRoute, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts new file mode 100644 index 0000000000000..2ee75e32af730 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { StreamDefinition, streamDefinitonSchema } from '../../../common/types'; +import { syncStream, readStream, checkStreamExists } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; + +export const editStreamRoute = createServerRoute({ + endpoint: 'PUT /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: z.object({ definition: streamDefinitonSchema }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + await validateStreamChildren( + scopedClusterClient, + params.path.id, + params.body.definition.children + ); + + const parentId = getParentId(params.path.id); + let parentDefinition: StreamDefinition | undefined; + + if (parentId) { + parentDefinition = await updateParentStream( + scopedClusterClient, + parentId, + params.path.id, + logger + ); + } + const streamDefinition = { ...params.body.definition }; + + await syncStream({ + scopedClusterClient, + definition: streamDefinition, + rootDefinition: parentDefinition, + logger, + }); + + for (const child of streamDefinition.children) { + const streamExists = await checkStreamExists({ + scopedClusterClient, + id: child.id, + }); + if (streamExists) { + continue; + } + // create empty streams for each child if they don't exist + const childDefinition = { + id: child.id, + children: [], + fields: [], + processing: [], + root: false, + }; + + await syncStream({ + scopedClusterClient, + definition: childDefinition, + logger, + }); + } + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + parentId: string, + id: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + if (!parentDefinition.children.some((child) => child.id === id)) { + // add the child to the parent stream with an empty condition for now + parentDefinition.children.push({ + id, + condition: undefined, + }); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + } + return parentDefinition; +} + +async function validateStreamChildren( + scopedClusterClient: IScopedClusterClient, + id: string, + children: Array<{ id: string }> +) { + try { + const { definition: oldDefinition } = await readStream({ + scopedClusterClient, + id, + }); + const oldChildren = oldDefinition.children.map((child) => child.id); + const newChildren = new Set(children.map((child) => child.id)); + if (oldChildren.some((child) => !newChildren.has(child))) { + throw new MalformedStreamId( + 'Cannot remove children from a stream, please delete the stream instead' + ); + } + } catch (e) { + // Ignore if the stream does not exist, but re-throw if it's another error + if (!(e instanceof DefinitionNotFound)) { + throw e; + } + } +} diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 1241434a975df..111cd870c41a4 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -8,8 +8,7 @@ import { z } from '@kbn/zod'; import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { bootstrapRootEntity } from '../../lib/streams/bootstrap_root_assets'; -import { createStream } from '../../lib/streams/stream_crud'; +import { syncStream } from '../../lib/streams/stream_crud'; import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; export const enableStreamsRoute = createServerRoute({ @@ -26,13 +25,10 @@ export const enableStreamsRoute = createServerRoute({ handler: async ({ request, response, logger, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); - await bootstrapRootEntity({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - }); - await createStream({ + await syncStream({ scopedClusterClient, definition: rootStreamDefinition, + logger, }); return response.ok({ body: { acknowledged: true } }); } catch (e) { diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index c019abd17b58d..ced96f465e3d3 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -13,10 +13,10 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { streamDefinitonSchema } from '../../../common/types'; -import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; -import { createStream, readStream } from '../../lib/streams/stream_crud'; +import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; +import { syncStream, readStream } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { isChildOf } from '../../lib/streams/helpers/hierarchy'; export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', @@ -32,7 +32,7 @@ export const forkStreamsRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: streamDefinitonSchema, + body: z.object({ stream: streamDefinitonSchema, condition: conditionSchema }), }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { @@ -47,20 +47,36 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - if (!params.body.id.startsWith(rootDefinition.id)) { + const childDefinition = { ...params.body.stream, root: false }; + + // check whether root stream has a child of the given name already + if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { + throw new MalformedStreamId( + `The stream with ID (${params.body.stream.id}) already exists as a child of the parent stream` + ); + } + + if (!isChildOf(rootDefinition, childDefinition)) { throw new MalformedStreamId( - `The ID (${params.body.id}) from the new stream must start with the parent's id (${rootDefinition.id})` + `The ID (${params.body.stream.id}) from the new stream must start with the parent's id (${rootDefinition.id}), followed by a dot and a name` ); } - await createStream({ + rootDefinition.children.push({ + id: params.body.stream.id, + condition: params.body.condition, + }); + + await syncStream({ scopedClusterClient, - definition: { ...params.body, forked_from: rootDefinition.id, root: false }, + definition: rootDefinition, + rootDefinition, + logger, }); - await bootstrapStream({ + await syncStream({ scopedClusterClient, - definition: params.body, + definition: params.body.stream, rootDefinition, logger, }); From 0e893453e3401a0d5b19c1e3b48252528ae61c8a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:27:08 +0100 Subject: [PATCH 05/18] add delete and listing endpoint --- .../component_templates/generate_layer.ts | 3 +- .../manage_component_templates.ts | 28 +++++ .../lib/streams/component_templates/name.ts | 10 ++ .../server/lib/streams/helpers/hierarchy.ts | 1 - .../generate_index_template.ts | 6 +- .../lib/streams/index_templates/name.ts | 10 ++ .../generate_ingest_pipeline.ts | 3 +- .../generate_reroute_pipeline.ts | 3 +- .../manage_ingest_pipelines.ts | 27 +++++ .../lib/streams/ingest_pipelines/name.ts | 14 +++ .../lib/streams/root_stream_definition.ts | 1 - .../streams/server/lib/streams/stream_crud.ts | 52 +++++++++ x-pack/plugins/streams/server/routes/index.ts | 4 + .../streams/server/routes/streams/delete.ts | 104 ++++++++++++++++++ .../streams/server/routes/streams/list.ts | 65 +++++++++++ 15 files changed, 324 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/delete.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/list.ts diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index f7398cb1304a8..4a39a78b38990 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -13,6 +13,7 @@ import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; import { logsSettings } from './logs_layer'; import { isRoot } from '../helpers/hierarchy'; +import { getComponentTemplateName } from './name'; export function generateLayer( id: string, @@ -25,7 +26,7 @@ export function generateLayer( }; }); return { - name: `${id}@stream.layer`, + name: getComponentTemplateName(id), template: { settings: isRoot(definition.id) ? logsSettings : {}, mappings: { diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts new file mode 100644 index 0000000000000..0eed31ecab086 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeleteComponentOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function deleteComponent({ esClient, name, logger }: DeleteComponentOptions) { + try { + await retryTransientEsErrors( + () => esClient.cluster.deleteComponentTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts new file mode 100644 index 0000000000000..6ea05b9a53b28 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getComponentTemplateName(id: string) { + return `${id}@stream.layer`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts index eeb69f89473a1..f31846bab36db 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -11,7 +11,6 @@ export function isDescendandOf(parent: StreamDefinition, child: StreamDefinition return child.id.startsWith(parent.id); } -// streams are named by dots: logs.a.b.c - a child means there is only one level of dots added export function isChildOf(parent: StreamDefinition, child: StreamDefinition) { return ( isDescendandOf(parent, child) && child.id.split('.').length === parent.id.split('.').length + 1 diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 14972ce8a6380..295c819a08a30 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -6,6 +6,8 @@ */ import { ASSET_VERSION } from '../../../../common/constants'; +import { getProcessingPipelineName } from '../ingest_pipelines/name'; +import { getIndexTemplateName } from './name'; export function generateIndexTemplate( id: string, @@ -13,7 +15,7 @@ export function generateIndexTemplate( ignoreMissing: string[] = [] ) { return { - name: `${id}@stream`, + name: getIndexTemplateName(id), index_patterns: [id], composed_of: [...composedOf, `${id}@stream.layer`], priority: 200, @@ -28,7 +30,7 @@ export function generateIndexTemplate( template: { settings: { index: { - default_pipeline: `${id}@stream.default-pipeline`, + default_pipeline: getProcessingPipelineName(id), }, }, }, diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts new file mode 100644 index 0000000000000..ec8ea5519a6b4 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getIndexTemplateName(id: string) { + return `${id}@stream`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 146671e79744f..eb09df8831304 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -10,10 +10,11 @@ import { ASSET_VERSION } from '../../../../common/constants'; import { conditionToPainless } from '../helpers/condition_to_painless'; import { logsDefaultPipelineProcessors } from './logs_default_pipeline'; import { isRoot } from '../helpers/hierarchy'; +import { getProcessingPipelineName } from './name'; export function generateIngestPipeline(id: string, definition: StreamDefinition) { return { - id: `${id}@stream.default-pipeline`, + id: getProcessingPipelineName(id), processors: [ ...(isRoot(definition.id) ? logsDefaultPipelineProcessors : []), ...definition.processing.map((processor) => { diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index b73dc61b148ed..9b46e0cf4ac92 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -8,6 +8,7 @@ import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; import { conditionToPainless } from '../helpers/condition_to_painless'; +import { getReroutePipelineName } from './name'; interface GenerateReroutePipelineParams { definition: StreamDefinition; @@ -15,7 +16,7 @@ interface GenerateReroutePipelineParams { export async function generateReroutePipeline({ definition }: GenerateReroutePipelineParams) { return { - id: `${definition.id}@stream.reroutes`, + id: getReroutePipelineName(definition.id), processors: definition.children.map((child) => { return { reroute: { diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts new file mode 100644 index 0000000000000..96449e142b68a --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeletePipelineOptions { + esClient: ElasticsearchClient; + id: string; + logger: Logger; +} + +export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipelineOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }), { + logger, + }); + } catch (error: any) { + logger.error(`Error deleting ingest pipeline: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts new file mode 100644 index 0000000000000..8d2a97ff3137f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getProcessingPipelineName(id: string) { + return `${id}@stream.processing`; +} + +export function getReroutePipelineName(id: string) { + return `${id}@stream.reroutes`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 3555146ccc008..4930876ae9923 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -9,7 +9,6 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { id: 'logs', - root: true, processing: [], children: [], fields: [ diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 5fe9e5f6de0df..53d5a86e7cd83 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -12,6 +12,7 @@ import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; import { + deleteTemplate, upsertComponent, upsertIngestPipeline, upsertTemplate, @@ -20,6 +21,11 @@ import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; import { generateIndexTemplate } from './index_templates/generate_index_template'; +import { deleteComponent } from './component_templates/manage_component_templates'; +import { deleteIngestPipeline } from './ingest_pipelines/manage_ingest_pipelines'; +import { getIndexTemplateName } from './index_templates/name'; +import { getComponentTemplateName } from './component_templates/name'; +import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -29,6 +35,39 @@ interface BaseParamsWithDefinition extends BaseParams { definition: StreamDefinition; } +interface DeleteStreamParams extends BaseParams { + id: string; + logger: Logger; +} + +export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { + await scopedClusterClient.asCurrentUser.delete({ + id, + index: STREAMS_INDEX, + refresh: 'wait_for', + }); + await deleteTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + name: getIndexTemplateName(id), + logger, + }); + await deleteComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + name: getComponentTemplateName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + id: getProcessingPipelineName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + id: getReroutePipelineName(id), + logger, + }); +} + async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, @@ -38,6 +77,19 @@ async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithD }); } +type ListStreamsParams = BaseParams; + +export async function listStreams({ scopedClusterClient }: ListStreamsParams) { + const response = await scopedClusterClient.asCurrentUser.search({ + index: STREAMS_INDEX, + size: 10000, + fields: ['id'], + _source: false, + }); + const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); + return definitions; +} + interface ReadStreamParams extends BaseParams { id: string; } diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index 544b60ccd404c..691b61e93aeab 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,9 +5,11 @@ * 2.0. */ +import { deleteStreamRoute } from './streams/delete'; import { editStreamRoute } from './streams/edit'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; +import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; export const StreamsRouteRepository = { @@ -15,6 +17,8 @@ export const StreamsRouteRepository = { ...forkStreamsRoute, ...readStreamRoute, ...editStreamRoute, + ...deleteStreamRoute, + ...listStreamsRoute, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts new file mode 100644 index 0000000000000..2176530a42518 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, deleteStreamObjects } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; + +export const deleteStreamRoute = createServerRoute({ + endpoint: 'DELETE /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + const parentId = getParentId(params.path.id); + if (!parentId) { + throw new MalformedStreamId('Cannot delete root stream'); + } + + await updateParentStream(scopedClusterClient, params.path.id, parentId, logger); + + await deleteStream(scopedClusterClient, params.path.id, logger); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function deleteStream(scopedClusterClient: IScopedClusterClient, id: string, logger: Logger) { + try { + const { definition } = await readStream({ scopedClusterClient, id }); + for (const child of definition.children) { + await deleteStream(scopedClusterClient, child.id, logger); + } + await deleteStreamObjects({ scopedClusterClient, id, logger }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + logger.debug(`Stream definition for ${id} not found.`); + } else { + throw e; + } + } +} + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + id: string, + parentId: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + parentDefinition.children = parentDefinition.children.filter((child) => child.id !== id); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + return parentDefinition; +} diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts new file mode 100644 index 0000000000000..37446788ad1de --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { listStreams } from '../../lib/streams/stream_crud'; + +export const listStreamsRoute = createServerRoute({ + endpoint: 'GET /api/streams 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + params: z.object({}), + handler: async ({ response, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const definitions = await listStreams({ scopedClusterClient }); + + const trees = asTrees(definitions); + + return response.ok({ body: { streams: trees } }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +interface ListStreamDefinition { + id: string; + children: ListStreamDefinition[]; +} + +function asTrees(definitions: Array<{ id: string[] }>) { + const trees: ListStreamDefinition[] = []; + definitions.forEach((definition) => { + const path = definition.id[0].split('.'); + let currentTree = trees; + path.forEach((_id, index) => { + const partialPath = path.slice(0, index + 1).join('.'); + const existingNode = currentTree.find((node) => node.id === partialPath); + if (existingNode) { + currentTree = existingNode.children; + } else { + const newNode = { id: partialPath, children: [] }; + currentTree.push(newNode); + currentTree = newNode.children; + } + }); + }); + return trees; +} From 9dd5fde2e977e7c265528e6aaaa5a87343cd354e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:34:45 +0100 Subject: [PATCH 06/18] cleanup --- .../manage_component_templates.ts | 19 ++++ .../index_templates/manage_index_templates.ts | 44 +++++++ .../manage_ingest_pipelines.ts | 21 ++++ .../streams/server/lib/streams/stream_crud.ts | 14 +-- x-pack/plugins/streams/server/plugin.ts | 6 - .../server/templates/components/base.ts | 60 ---------- .../templates/manage_index_templates.ts | 107 ------------------ .../templates/streams_index_template.ts | 43 ------- 8 files changed, 90 insertions(+), 224 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts delete mode 100644 x-pack/plugins/streams/server/templates/components/base.ts delete mode 100644 x-pack/plugins/streams/server/templates/manage_index_templates.ts delete mode 100644 x-pack/plugins/streams/server/templates/streams_index_template.ts diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts index 0eed31ecab086..a7d707a4ce42a 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts @@ -7,6 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { retryTransientEsErrors } from '../helpers/retry'; interface DeleteComponentOptions { @@ -15,6 +16,12 @@ interface DeleteComponentOptions { logger: Logger; } +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + export async function deleteComponent({ esClient, name, logger }: DeleteComponentOptions) { try { await retryTransientEsErrors( @@ -26,3 +33,15 @@ export async function deleteComponent({ esClient, name, logger }: DeleteComponen throw error; } } + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); + logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); + } catch (error: any) { + logger.error(`Error updating component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts new file mode 100644 index 0000000000000..9383e698b3436 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface TemplateManagementOptions { + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; + logger: Logger; +} + +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts index 96449e142b68a..467e2efb48f0d 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts @@ -7,6 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { IngestPutPipelineRequest } from '@elastic/elasticsearch/lib/api/types'; import { retryTransientEsErrors } from '../helpers/retry'; interface DeletePipelineOptions { @@ -15,6 +16,12 @@ interface DeletePipelineOptions { logger: Logger; } +interface PipelineManagementOptions { + esClient: ElasticsearchClient; + pipeline: IngestPutPipelineRequest; + logger: Logger; +} + export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipelineOptions) { try { await retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }), { @@ -25,3 +32,17 @@ export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipel throw error; } } + +export async function upsertIngestPipeline({ + esClient, + pipeline, + logger, +}: PipelineManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 53d5a86e7cd83..b557714da1682 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -11,21 +11,19 @@ import { Logger } from '@kbn/logging'; import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; -import { - deleteTemplate, - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; +import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; import { generateIndexTemplate } from './index_templates/generate_index_template'; -import { deleteComponent } from './component_templates/manage_component_templates'; -import { deleteIngestPipeline } from './ingest_pipelines/manage_ingest_pipelines'; +import { deleteComponent, upsertComponent } from './component_templates/manage_component_templates'; import { getIndexTemplateName } from './index_templates/name'; import { getComponentTemplateName } from './component_templates/name'; import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; +import { + deleteIngestPipeline, + upsertIngestPipeline, +} from './ingest_pipelines/manage_ingest_pipelines'; interface BaseParams { scopedClusterClient: IScopedClusterClient; diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 16473918e2cfa..951c1e3751d72 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -17,7 +17,6 @@ import { } from '@kbn/core/server'; import { registerRoutes } from '@kbn/server-route-repository'; import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; -import { installStreamsTemplates } from './templates/manage_index_templates'; import { StreamsRouteRepository } from './routes'; import { RouteDependencies } from './routes/types'; import { @@ -118,11 +117,6 @@ export class StreamsPlugin this.server.taskManager = plugins.taskManager; } - const esClient = core.elasticsearch.client.asInternalUser; - installStreamsTemplates({ esClient, logger: this.logger }).catch((err) => - this.logger.error(err) - ); - return {}; } diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts deleted file mode 100644 index fac220fb8be1f..0000000000000 --- a/x-pack/plugins/streams/server/templates/components/base.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; - -export const STREAMS_BASE_COMPONENT = 'streams@mappings'; - -export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: STREAMS_BASE_COMPONENT, - _meta: { - description: 'Component template for the Stream Entities Manager data set', - managed: true, - }, - template: { - mappings: { - properties: { - labels: { - type: 'object', - }, - tags: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - dataset: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - root: { - type: 'boolean', - }, - forked_from: { - ignore_above: 1024, - type: 'keyword', - }, - condition: { - type: 'object', - }, - event: { - properties: { - ingested: { - type: 'date', - }, - }, - }, - }, - }, - }, -}; diff --git a/x-pack/plugins/streams/server/templates/manage_index_templates.ts b/x-pack/plugins/streams/server/templates/manage_index_templates.ts deleted file mode 100644 index f562c4dfe4183..0000000000000 --- a/x-pack/plugins/streams/server/templates/manage_index_templates.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - ClusterPutComponentTemplateRequest, - IndicesPutIndexTemplateRequest, - IngestPutPipelineRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { BaseComponentTemplateConfig } from './components/base'; -import { retryTransientEsErrors } from '../lib/streams/helpers/retry'; -import { streamsIndexTemplate } from './streams_index_template'; - -interface TemplateManagementOptions { - esClient: ElasticsearchClient; - template: IndicesPutIndexTemplateRequest; - logger: Logger; -} - -interface PipelineManagementOptions { - esClient: ElasticsearchClient; - pipeline: IngestPutPipelineRequest; - logger: Logger; -} - -interface ComponentManagementOptions { - esClient: ElasticsearchClient; - component: ClusterPutComponentTemplateRequest; - logger: Logger; -} - -export const installStreamsTemplates = async ({ - esClient, - logger, -}: { - esClient: ElasticsearchClient; - logger: Logger; -}) => { - await upsertComponent({ - esClient, - logger, - component: BaseComponentTemplateConfig, - }); - await upsertTemplate({ - esClient, - logger, - template: streamsIndexTemplate, - }); -}; - -interface DeleteTemplateOptions { - esClient: ElasticsearchClient; - name: string; - logger: Logger; -} - -export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); - logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); - } catch (error: any) { - logger.error(`Error updating index template: ${error.message}`); - throw error; - } -} - -export async function upsertIngestPipeline({ - esClient, - pipeline, - logger, -}: PipelineManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); - logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); - } catch (error: any) { - logger.error(`Error updating index template: ${error.message}`); - throw error; - } -} - -export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { - try { - await retryTransientEsErrors( - () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), - { logger } - ); - } catch (error: any) { - logger.error(`Error deleting index template: ${error.message}`); - throw error; - } -} - -export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { - logger, - }); - logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); - } catch (error: any) { - logger.error(`Error updating component template: ${error.message}`); - throw error; - } -} diff --git a/x-pack/plugins/streams/server/templates/streams_index_template.ts b/x-pack/plugins/streams/server/templates/streams_index_template.ts deleted file mode 100644 index 0e843b2289cf3..0000000000000 --- a/x-pack/plugins/streams/server/templates/streams_index_template.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { STREAMS_BASE_COMPONENT } from './components/base'; -import { STREAMS_INDEX } from '../../common/constants'; - -export const streamsIndexTemplate: IndicesPutIndexTemplateRequest = { - name: 'stream-entities', - _meta: { - description: - 'Index template for indices managed by the Streams framework for the instance dataset', - ecs_version: '8.0.0', - managed: true, - managed_by: 'streams', - }, - composed_of: [STREAMS_BASE_COMPONENT], - index_patterns: [STREAMS_INDEX], - priority: 200, - template: { - mappings: { - _meta: { - version: '1.6.0', - }, - date_detection: false, - dynamic: false, - }, - settings: { - index: { - codec: 'best_compression', - mapping: { - total_fields: { - limit: 2000, - }, - }, - }, - }, - }, -}; From 7400a3bbb356b0d71b98959b995e125f83a75c46 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:45:15 +0100 Subject: [PATCH 07/18] add resync endpoint --- .../streams/server/lib/streams/stream_crud.ts | 1 + x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/resync.ts | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 x-pack/plugins/streams/server/routes/streams/resync.ts diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index b557714da1682..7ffadf552fcbc 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -83,6 +83,7 @@ export async function listStreams({ scopedClusterClient }: ListStreamsParams) { size: 10000, fields: ['id'], _source: false, + sort: [{ id: 'asc' }], }); const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); return definitions; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index 691b61e93aeab..6fc734d3371b4 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -11,9 +11,11 @@ import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; +import { resyncStreamsRoute } from './streams/resync'; export const StreamsRouteRepository = { ...enableStreamsRoute, + ...resyncStreamsRoute, ...forkStreamsRoute, ...readStreamRoute, ...editStreamRoute, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts new file mode 100644 index 0000000000000..e2888328e9fba --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, listStreams } from '../../lib/streams/stream_crud'; + +export const resyncStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_resync 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({}), + handler: async ({ response, logger, request, getScopedClients }) => { + const { scopedClusterClient } = await getScopedClients({ request }); + + const streams = await listStreams({ scopedClusterClient }); + + for (const stream of streams) { + const { definition } = await readStream({ + scopedClusterClient, + id: stream.id[0], + }); + await syncStream({ + scopedClusterClient, + definition, + logger, + }); + } + + return response.ok({}); + }, +}); From 9d372194f7d839c095d164815c0560673c3d20fb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 10 Nov 2024 21:00:19 +0100 Subject: [PATCH 08/18] more fixes --- x-pack/plugins/streams/common/types.ts | 23 ++-- .../lib/streams/errors/malformed_children.ts | 13 ++ .../lib/streams/errors/malformed_fields.ts | 13 ++ .../server/lib/streams/helpers/hierarchy.ts | 5 + .../generate_index_template.ts | 19 +-- .../lib/streams/internal_stream_mapping.ts | 35 ++++++ .../streams/server/lib/streams/stream_crud.ts | 117 +++++++++++++++--- .../streams/server/routes/streams/edit.ts | 28 +++-- .../streams/server/routes/streams/enable.ts | 2 + .../streams/server/routes/streams/fork.ts | 10 +- .../streams/server/routes/streams/read.ts | 16 ++- 11 files changed, 224 insertions(+), 57 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 0019eb8fee054..9e35e50581782 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -65,22 +65,7 @@ export const fieldDefinitionSchema = z.object({ export type FieldDefinition = z.infer; -/** - * Example of a "root" stream - * { - * "id": "logs", - * } - * - * Example of a forked stream - * { - * "id": "logs.nginx", - * "condition": { field: 'log.logger, operator: 'eq', value": "nginx_proxy" } - * "forked_from": "logs" - * } - */ - -export const streamDefinitonSchema = z.object({ - id: z.string(), +export const streamWithoutIdDefinitonSchema = z.object({ processing: z.array(processingDefinitionSchema).default([]), fields: z.array(fieldDefinitionSchema).default([]), children: z @@ -93,4 +78,10 @@ export const streamDefinitonSchema = z.object({ .default([]), }); +export type StreamWithoutIdDefinition = z.infer; + +export const streamDefinitonSchema = streamWithoutIdDefinitonSchema.extend({ + id: z.string(), +}); + export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts new file mode 100644 index 0000000000000..699c4cdd5b1ef --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedChildren extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedChildren'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts new file mode 100644 index 0000000000000..b8f7ac1392610 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedFields extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedFields'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts index f31846bab36db..6f1cd308f3c3d 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -28,3 +28,8 @@ export function getParentId(id: string) { export function isRoot(id: string) { return id.split('.').length === 1; } + +export function getAncestors(id: string) { + const parts = id.split('.'); + return parts.slice(0, parts.length - 1).map((_, index) => parts.slice(0, index + 1).join('.')); +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 295c819a08a30..8f7773c455c4d 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -9,15 +9,19 @@ import { ASSET_VERSION } from '../../../../common/constants'; import { getProcessingPipelineName } from '../ingest_pipelines/name'; import { getIndexTemplateName } from './name'; -export function generateIndexTemplate( - id: string, - composedOf: string[] = [], - ignoreMissing: string[] = [] -) { +export function generateIndexTemplate(id: string) { + const composedOf = id.split('.').reduce((acc, _, index, array) => { + if (index === array.length - 1) { + return acc; + } + const parent = array.slice(0, index + 1).join('.'); + return [...acc, `${parent}@stream.layer`]; + }, [] as string[]); + return { name: getIndexTemplateName(id), index_patterns: [id], - composed_of: [...composedOf, `${id}@stream.layer`], + composed_of: composedOf, priority: 200, version: ASSET_VERSION, _meta: { @@ -35,6 +39,7 @@ export function generateIndexTemplate( }, }, allow_auto_create: true, - ignore_missing_component_templates: [...ignoreMissing, `${id}@stream.layer`], + // ignore missing component templates to be more robust against out-of-order syncs + ignore_missing_component_templates: composedOf, }; } diff --git a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts new file mode 100644 index 0000000000000..2428cc798f006 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { STREAMS_INDEX } from '../../../common/constants'; + +export function createStreamsIndex(scopedClusterClient: IScopedClusterClient) { + return scopedClusterClient.asCurrentUser.indices.create({ + index: STREAMS_INDEX, + mappings: { + dynamic: 'strict', + properties: { + processing: { + type: 'object', + enabled: false, + }, + fields: { + type: 'object', + enabled: false, + }, + children: { + type: 'object', + enabled: false, + }, + id: { + type: 'keyword', + }, + }, + }, + }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 7ffadf552fcbc..6b29bd830ab14 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -8,7 +8,7 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { get } from 'lodash'; import { Logger } from '@kbn/logging'; -import { StreamDefinition } from '../../../common/types'; +import { FieldDefinition, StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; @@ -24,6 +24,8 @@ import { deleteIngestPipeline, upsertIngestPipeline, } from './ingest_pipelines/manage_ingest_pipelines'; +import { getAncestors } from './helpers/hierarchy'; +import { MalformedFields } from './errors/malformed_fields'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -66,7 +68,7 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D }); } -async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { +async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, index: STREAMS_INDEX, @@ -111,6 +113,95 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) } } +interface ReadAncestorsParams extends BaseParams { + id: string; +} + +export async function readAncestors({ id, scopedClusterClient }: ReadAncestorsParams) { + const ancestorIds = getAncestors(id); + + return await Promise.all( + ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId })) + ); +} + +interface ReadDescendantsParams extends BaseParams { + id: string; +} + +export async function readDescendants({ id, scopedClusterClient }: ReadDescendantsParams) { + const response = await scopedClusterClient.asCurrentUser.search({ + index: STREAMS_INDEX, + size: 10000, + body: { + query: { + bool: { + filter: { + prefix: { + id, + }, + }, + must_not: { + term: { + id, + }, + }, + }, + }, + }, + }); + return response.hits.hits.map((hit) => hit._source as StreamDefinition); +} + +export async function validateAncestorFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const ancestors = await readAncestors({ + id, + scopedClusterClient, + }); + for (const ancestor of ancestors) { + for (const field of fields) { + if ( + ancestor.definition.fields.some( + (ancestorField) => ancestorField.type !== field.type && ancestorField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the parent stream ${ancestor.definition.id}` + ); + } + } + } +} + +export async function validateDescendantFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const descendants = await readDescendants({ + id, + scopedClusterClient, + }); + for (const descendant of descendants) { + for (const field of fields) { + if ( + descendant.fields.some( + (descendantField) => + descendantField.type !== field.type && descendantField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the child stream ${descendant.id}` + ); + } + } + } +} + export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamParams) { try { await readStream({ id, scopedClusterClient }); @@ -167,7 +258,7 @@ export async function syncStream({ rootDefinition, logger, }: SyncStreamParams) { - await upsertStream({ + await upsertInternalStream({ scopedClusterClient, definition, }); @@ -189,11 +280,12 @@ export async function syncStream({ logger, pipeline: reroutePipeline, }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id), + }); if (rootDefinition) { - const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ - scopedClusterClient, - definition: rootDefinition, - }); const parentReroutePipeline = await generateReroutePipeline({ definition: rootDefinition, }); @@ -202,16 +294,5 @@ export async function syncStream({ logger, pipeline: parentReroutePipeline, }); - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), - }); - } else { - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id), - }); } } diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 2ee75e32af730..a3c41eef5a7a3 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -15,10 +15,17 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { StreamDefinition, streamDefinitonSchema } from '../../../common/types'; -import { syncStream, readStream, checkStreamExists } from '../../lib/streams/stream_crud'; +import { StreamDefinition, streamWithoutIdDefinitonSchema } from '../../../common/types'; +import { + syncStream, + readStream, + checkStreamExists, + validateAncestorFields, + validateDescendantFields, +} from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { getParentId } from '../../lib/streams/helpers/hierarchy'; +import { MalformedChildren } from '../../lib/streams/errors/malformed_children'; export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id} 2023-10-31', @@ -34,17 +41,15 @@ export const editStreamRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: z.object({ definition: streamDefinitonSchema }), + body: streamWithoutIdDefinitonSchema, }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); - await validateStreamChildren( - scopedClusterClient, - params.path.id, - params.body.definition.children - ); + await validateStreamChildren(scopedClusterClient, params.path.id, params.body.children); + await validateAncestorFields(scopedClusterClient, params.path.id, params.body.fields); + await validateDescendantFields(scopedClusterClient, params.path.id, params.body.fields); const parentId = getParentId(params.path.id); let parentDefinition: StreamDefinition | undefined; @@ -57,11 +62,11 @@ export const editStreamRoute = createServerRoute({ logger ); } - const streamDefinition = { ...params.body.definition }; + const streamDefinition = { ...params.body }; await syncStream({ scopedClusterClient, - definition: streamDefinition, + definition: { ...streamDefinition, id: params.path.id }, rootDefinition: parentDefinition, logger, }); @@ -80,7 +85,6 @@ export const editStreamRoute = createServerRoute({ children: [], fields: [], processing: [], - root: false, }; await syncStream({ @@ -149,7 +153,7 @@ async function validateStreamChildren( const oldChildren = oldDefinition.children.map((child) => child.id); const newChildren = new Set(children.map((child) => child.id)); if (oldChildren.some((child) => !newChildren.has(child))) { - throw new MalformedStreamId( + throw new MalformedChildren( 'Cannot remove children from a stream, please delete the stream instead' ); } diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 111cd870c41a4..3657ef04fbbb7 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -10,6 +10,7 @@ import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; import { syncStream } from '../../lib/streams/stream_crud'; import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; +import { createStreamsIndex } from '../../lib/streams/internal_stream_mapping'; export const enableStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_enable 2023-10-31', @@ -25,6 +26,7 @@ export const enableStreamsRoute = createServerRoute({ handler: async ({ request, response, logger, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); + await createStreamsIndex(scopedClusterClient); await syncStream({ scopedClusterClient, definition: rootStreamDefinition, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index ced96f465e3d3..1d626705d304a 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -14,7 +14,7 @@ import { } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; -import { syncStream, readStream } from '../../lib/streams/stream_crud'; +import { syncStream, readStream, validateAncestorFields } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { isChildOf } from '../../lib/streams/helpers/hierarchy'; @@ -47,7 +47,7 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - const childDefinition = { ...params.body.stream, root: false }; + const childDefinition = { ...params.body.stream }; // check whether root stream has a child of the given name already if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { @@ -62,6 +62,12 @@ export const forkStreamsRoute = createServerRoute({ ); } + await validateAncestorFields( + scopedClusterClient, + params.body.stream.id, + params.body.stream.fields + ); + rootDefinition.children.push({ id: params.body.stream.id, condition: params.body.condition, diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 061522787de52..64914a3d9c741 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -8,7 +8,7 @@ import { z } from '@kbn/zod'; import { createServerRoute } from '../create_server_route'; import { DefinitionNotFound } from '../../lib/streams/errors'; -import { readStream } from '../../lib/streams/stream_crud'; +import { readAncestors, readStream } from '../../lib/streams/stream_crud'; export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id} 2023-10-31', @@ -31,7 +31,19 @@ export const readStreamRoute = createServerRoute({ id: params.path.id, }); - return response.ok({ body: streamEntity }); + const ancestors = await readAncestors({ + id: streamEntity.definition.id, + scopedClusterClient, + }); + + const body = { + ...streamEntity.definition, + inheritedFields: ancestors.flatMap(({ definition: { id, fields } }) => + fields.map((field) => ({ ...field, from: id })) + ), + }; + + return response.ok({ body }); } catch (e) { if (e instanceof DefinitionNotFound) { return response.notFound({ body: e }); From d622fa1e29826e0d51f8db4a4fbe985bc4ec5ec6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 11 Nov 2024 14:51:00 +0100 Subject: [PATCH 09/18] more adjustments --- x-pack/plugins/streams/common/constants.ts | 2 +- .../component_templates/generate_layer.ts | 1 + .../data_streams/manage_data_streams.ts | 93 ++++++++++++++++++ .../generate_index_template.ts | 3 - .../lib/streams/internal_stream_mapping.ts | 2 +- .../streams/server/lib/streams/stream_crud.ts | 98 ++++++++----------- x-pack/plugins/streams/server/plugin.ts | 2 +- .../streams/server/routes/streams/delete.ts | 2 +- .../streams/server/routes/streams/edit.ts | 2 +- .../streams/server/routes/streams/enable.ts | 2 +- .../streams/server/routes/streams/fork.ts | 2 +- .../streams/server/routes/streams/resync.ts | 2 +- 12 files changed, 145 insertions(+), 66 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts diff --git a/x-pack/plugins/streams/common/constants.ts b/x-pack/plugins/streams/common/constants.ts index acc66e9286f87..d7595990ded6e 100644 --- a/x-pack/plugins/streams/common/constants.ts +++ b/x-pack/plugins/streams/common/constants.ts @@ -6,4 +6,4 @@ */ export const ASSET_VERSION = 1; -export const STREAMS_INDEX = '.streams'; +export const STREAMS_INDEX = '.kibana_streams'; diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index 4a39a78b38990..82c89c9ab9171 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -30,6 +30,7 @@ export function generateLayer( template: { settings: isRoot(definition.id) ? logsSettings : {}, mappings: { + subobjects: false, properties, }, }, diff --git a/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts new file mode 100644 index 0000000000000..812739db56c73 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DataStreamManagementOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface DeleteDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface RolloverDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertDataStream({ esClient, name, logger }: DataStreamManagementOptions) { + const dataStreamExists = await esClient.indices.exists({ index: name }); + if (dataStreamExists) { + return; + } + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + logger.debug(() => `Installed data stream: ${name}`); + } catch (error: any) { + logger.error(`Error creating data stream: ${error.message}`); + throw error; + } +} + +export async function deleteDataStream({ esClient, name, logger }: DeleteDataStreamOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteDataStream({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting data stream: ${error.message}`); + throw error; + } +} + +export async function rolloverDataStreamIfNecessary({ + esClient, + name, + logger, +}: RolloverDataStreamOptions) { + const dataStreams = await esClient.indices.getDataStream({ name: `${name},${name}.*` }); + for (const dataStream of dataStreams.data_streams) { + const currentMappings = + Object.values( + await esClient.indices.getMapping({ + index: dataStream.indices.at(-1)?.index_name, + }) + )[0].mappings.properties || {}; + const simulatedIndex = await esClient.indices.simulateIndexTemplate({ name: dataStream.name }); + const simulatedMappings = simulatedIndex.template.mappings.properties || {}; + + // check whether the same fields and same types are listed (don't check for other mapping attributes) + const isDifferent = + Object.values(simulatedMappings).length !== Object.values(currentMappings).length || + Object.entries(simulatedMappings || {}).some(([fieldName, { type }]) => { + const currentType = currentMappings[fieldName]?.type; + return currentType !== type; + }); + + if (!isDifferent) { + continue; + } + + try { + await retryTransientEsErrors(() => esClient.indices.rollover({ alias: dataStream.name }), { + logger, + }); + logger.debug(() => `Rolled over data stream: ${dataStream.name}`); + } catch (error: any) { + logger.error(`Error rolling over data stream: ${error.message}`); + throw error; + } + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 8f7773c455c4d..7a16534a618da 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -11,9 +11,6 @@ import { getIndexTemplateName } from './name'; export function generateIndexTemplate(id: string) { const composedOf = id.split('.').reduce((acc, _, index, array) => { - if (index === array.length - 1) { - return acc; - } const parent = array.slice(0, index + 1).join('.'); return [...acc, `${parent}@stream.layer`]; }, [] as string[]); diff --git a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts index 2428cc798f006..8e88eeef8cd84 100644 --- a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts +++ b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts @@ -9,7 +9,7 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { STREAMS_INDEX } from '../../../common/constants'; export function createStreamsIndex(scopedClusterClient: IScopedClusterClient) { - return scopedClusterClient.asCurrentUser.indices.create({ + return scopedClusterClient.asInternalUser.indices.create({ index: STREAMS_INDEX, mappings: { dynamic: 'strict', diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 6b29bd830ab14..b5021b2aebeb4 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -6,11 +6,10 @@ */ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; -import { get } from 'lodash'; import { Logger } from '@kbn/logging'; import { FieldDefinition, StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; -import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { DefinitionNotFound } from './errors'; import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; @@ -26,6 +25,11 @@ import { } from './ingest_pipelines/manage_ingest_pipelines'; import { getAncestors } from './helpers/hierarchy'; import { MalformedFields } from './errors/malformed_fields'; +import { + deleteDataStream, + rolloverDataStreamIfNecessary, + upsertDataStream, +} from './data_streams/manage_data_streams'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -41,35 +45,40 @@ interface DeleteStreamParams extends BaseParams { } export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { - await scopedClusterClient.asCurrentUser.delete({ - id, - index: STREAMS_INDEX, - refresh: 'wait_for', - }); await deleteTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, name: getIndexTemplateName(id), logger, }); await deleteComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, name: getComponentTemplateName(id), logger, }); await deleteIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, id: getProcessingPipelineName(id), logger, }); await deleteIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, id: getReroutePipelineName(id), logger, }); + await deleteDataStream({ + esClient: scopedClusterClient.asCurrentUser, + name: id, + logger, + }); + await scopedClusterClient.asInternalUser.delete({ + id, + index: STREAMS_INDEX, + refresh: 'wait_for', + }); } async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { - return scopedClusterClient.asCurrentUser.index({ + return scopedClusterClient.asInternalUser.index({ id: definition.id, index: STREAMS_INDEX, document: definition, @@ -80,7 +89,7 @@ async function upsertInternalStream({ definition, scopedClusterClient }: BasePar type ListStreamsParams = BaseParams; export async function listStreams({ scopedClusterClient }: ListStreamsParams) { - const response = await scopedClusterClient.asCurrentUser.search({ + const response = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, fields: ['id'], @@ -97,7 +106,7 @@ interface ReadStreamParams extends BaseParams { export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { try { - const response = await scopedClusterClient.asCurrentUser.get({ + const response = await scopedClusterClient.asInternalUser.get({ id, index: STREAMS_INDEX, }); @@ -130,7 +139,7 @@ interface ReadDescendantsParams extends BaseParams { } export async function readDescendants({ id, scopedClusterClient }: ReadDescendantsParams) { - const response = await scopedClusterClient.asCurrentUser.search({ + const response = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, body: { @@ -214,37 +223,6 @@ export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamP } } -export async function readIndexTemplate({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ - name: `${definition.id}@stream`, - }); - const indexTemplate = response.index_templates.find( - (doc) => doc.name === `${definition.id}@stream` - ); - if (!indexTemplate) { - throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); - } - return indexTemplate; -} - -export async function getIndexTemplateComponents({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - return { - composedOf: indexTemplate.index_template.composed_of, - ignoreMissing: get( - indexTemplate, - 'index_template.ignore_missing_component_templates', - [] - ) as string[], - }; -} - interface SyncStreamParams { scopedClusterClient: IScopedClusterClient; definition: StreamDefinition; @@ -258,17 +236,13 @@ export async function syncStream({ rootDefinition, logger, }: SyncStreamParams) { - await upsertInternalStream({ - scopedClusterClient, - definition, - }); await upsertComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, component: generateLayer(definition.id, definition), }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: generateIngestPipeline(definition.id, definition), }); @@ -276,12 +250,12 @@ export async function syncStream({ definition, }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: reroutePipeline, }); await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, template: generateIndexTemplate(definition.id), }); @@ -290,9 +264,23 @@ export async function syncStream({ definition: rootDefinition, }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: parentReroutePipeline, }); } + await upsertDataStream({ + esClient: scopedClusterClient.asCurrentUser, + logger, + name: definition.id, + }); + await upsertInternalStream({ + scopedClusterClient, + definition, + }); + await rolloverDataStreamIfNecessary({ + esClient: scopedClusterClient.asCurrentUser, + name: definition.id, + logger, + }); } diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 951c1e3751d72..478c77c6ca691 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -75,7 +75,7 @@ export class StreamsPlugin read: [], }, ui: ['read', 'write'], - api: ['streams_enable', 'streams_fork', 'streams_read'], + api: ['streams_write', 'streams_read'], }, read: { app: ['streams', 'kibana'], diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index 2176530a42518..cea5275e9b409 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -25,7 +25,7 @@ export const deleteStreamRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index a3c41eef5a7a3..5c027a9985a07 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -33,7 +33,7 @@ export const editStreamRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 3657ef04fbbb7..3e122b855575d 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -19,7 +19,7 @@ export const enableStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_enable'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1d626705d304a..f0ebb2a03ce52 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -24,7 +24,7 @@ export const forkStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index e2888328e9fba..badba1fe90346 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -15,7 +15,7 @@ export const resyncStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, From fbc6c2a19e72d2e603c54e19f08232c837d708ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 11 Nov 2024 16:00:41 +0100 Subject: [PATCH 10/18] some more fixes --- x-pack/plugins/streams/common/types.ts | 6 +++++- .../server/lib/streams/root_stream_definition.ts | 2 +- .../plugins/streams/server/lib/streams/stream_crud.ts | 10 +++++----- x-pack/plugins/streams/server/routes/streams/fork.ts | 8 ++++---- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 9e35e50581782..6cdb2f923f6f4 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -60,7 +60,7 @@ export type ProcessingDefinition = z.infer; export const fieldDefinitionSchema = z.object({ name: z.string(), - type: z.enum(['keyword', 'text', 'long', 'double', 'date', 'boolean', 'ip']), + type: z.enum(['keyword', 'match_only_text', 'long', 'double', 'date', 'boolean', 'ip']), }); export type FieldDefinition = z.infer; @@ -85,3 +85,7 @@ export const streamDefinitonSchema = streamWithoutIdDefinitonSchema.extend({ }); export type StreamDefinition = z.infer; + +export const streamDefinitonWithoutChildrenSchema = streamDefinitonSchema.omit({ children: true }); + +export type StreamWithoutChildrenDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 4930876ae9923..2b7deed877309 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -18,7 +18,7 @@ export const rootStreamDefinition: StreamDefinition = { }, { name: 'message', - type: 'text', + type: 'match_only_text', }, { name: 'host.name', diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index b5021b2aebeb4..78a126905d9a4 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -45,6 +45,11 @@ interface DeleteStreamParams extends BaseParams { } export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { + await deleteDataStream({ + esClient: scopedClusterClient.asCurrentUser, + name: id, + logger, + }); await deleteTemplate({ esClient: scopedClusterClient.asCurrentUser, name: getIndexTemplateName(id), @@ -65,11 +70,6 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D id: getReroutePipelineName(id), logger, }); - await deleteDataStream({ - esClient: scopedClusterClient.asCurrentUser, - name: id, - logger, - }); await scopedClusterClient.asInternalUser.delete({ id, index: STREAMS_INDEX, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index f0ebb2a03ce52..1ff507e530065 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -13,7 +13,7 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; +import { conditionSchema, streamDefinitonWithoutChildrenSchema } from '../../../common/types'; import { syncStream, readStream, validateAncestorFields } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { isChildOf } from '../../lib/streams/helpers/hierarchy'; @@ -32,7 +32,7 @@ export const forkStreamsRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: z.object({ stream: streamDefinitonSchema, condition: conditionSchema }), + body: z.object({ stream: streamDefinitonWithoutChildrenSchema, condition: conditionSchema }), }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { @@ -47,7 +47,7 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - const childDefinition = { ...params.body.stream }; + const childDefinition = { ...params.body.stream, children: [] }; // check whether root stream has a child of the given name already if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { @@ -82,7 +82,7 @@ export const forkStreamsRoute = createServerRoute({ await syncStream({ scopedClusterClient, - definition: params.body.stream, + definition: childDefinition, rootDefinition, logger, }); From e810f537bd60ec172d6fd14d3165c31aab21c1ef Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 08:32:38 +0000 Subject: [PATCH 11/18] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index abc6749d52ea9..85999705f2a64 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -966,6 +966,7 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops +x-pack/plugins/streams @elastic/obs-entities x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index b6ba24df78976..71ab26400f496 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -897,6 +897,10 @@ routes, etc. |The stack_connectors plugin provides connector types shipped with Kibana, built on top of the framework provided in the actions plugin. +|{kib-repo}blob/{branch}/x-pack/plugins/streams/README.md[streams] +|This plugin provides an interface to manage streams + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/synthetics/README.md[synthetics] |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening in their infrastructure. From e93ac173c857b4183e2f06fcd3284c082a51ccbd Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 08:44:23 +0000 Subject: [PATCH 12/18] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/streams/tsconfig.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index 98386b54ca9ea..e0ceaf8197fa6 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -14,28 +14,19 @@ "target/**/*" ], "kbn_references": [ - "@kbn/entities-schema", "@kbn/config-schema", "@kbn/core", - "@kbn/server-route-repository-client", "@kbn/logging", "@kbn/core-plugins-server", "@kbn/core-http-server", "@kbn/security-plugin", - "@kbn/rison", - "@kbn/es-query", - "@kbn/core-elasticsearch-client-server-mocks", - "@kbn/core-saved-objects-api-server-mocks", - "@kbn/logging-mocks", "@kbn/core-saved-objects-api-server", "@kbn/core-elasticsearch-server", "@kbn/task-manager-plugin", - "@kbn/datemath", "@kbn/server-route-repository", "@kbn/zod", - "@kbn/zod-helpers", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/alerting-plugin" + "@kbn/features-plugin" ] } From cf9f349bf2e893aa6152ca325c8ac75e6d8069ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 10:36:51 +0100 Subject: [PATCH 13/18] add limits --- packages/kbn-optimizer/limits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index df8a077e844f6..bdfb27369a372 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -159,6 +159,7 @@ pageLoadAssetSize: spaces: 57868 stackAlerts: 58316 stackConnectors: 67227 + streams: 16742 synthetics: 55971 telemetry: 51957 telemetryManagementSection: 38586 From 34f3c44fe707d860c77f038fb148fe40b05463ae Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:36:45 +0100 Subject: [PATCH 14/18] fix permissions --- x-pack/plugins/streams/server/plugin.ts | 23 +------------------ .../streams/server/routes/streams/delete.ts | 7 +++++- .../streams/server/routes/streams/edit.ts | 7 +++++- .../streams/server/routes/streams/enable.ts | 7 +++++- .../streams/server/routes/streams/fork.ts | 7 +++++- .../streams/server/routes/streams/list.ts | 7 +++++- .../streams/server/routes/streams/read.ts | 7 +++++- .../streams/server/routes/streams/resync.ts | 7 +++++- 8 files changed, 43 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 478c77c6ca691..2d9f911440b1a 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -66,28 +66,7 @@ export class StreamsPlugin app: ['streams', 'kibana'], catalogue: ['streams', 'observability'], category: DEFAULT_APP_CATEGORIES.observability, - privileges: { - all: { - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - savedObject: { - all: [], - read: [], - }, - ui: ['read', 'write'], - api: ['streams_write', 'streams_read'], - }, - read: { - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - api: ['streams_read'], - ui: ['read'], - savedObject: { - all: [], - read: [], - }, - }, - }, + privileges: null, }); registerRoutes({ diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index cea5275e9b409..3820975dbe16a 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -23,9 +23,14 @@ export const deleteStreamRoute = createServerRoute({ endpoint: 'DELETE /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 5c027a9985a07..b82b4d54044da 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -31,9 +31,14 @@ export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 3e122b855575d..27d8929b28e50 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -17,9 +17,14 @@ export const enableStreamsRoute = createServerRoute({ params: z.object({}), options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1ff507e530065..44f4052878003 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -22,9 +22,14 @@ export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index 37446788ad1de..2e4f13a89bb41 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -14,9 +14,14 @@ export const listStreamsRoute = createServerRoute({ endpoint: 'GET /api/streams 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_read'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 64914a3d9c741..5ea2aaf5f2542 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -14,9 +14,14 @@ export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_read'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index badba1fe90346..2365252ab00e6 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -13,9 +13,14 @@ export const resyncStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_resync 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, From 0b7de2475f6f87ea4f70b950d05c16b2163ac15d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:48:58 +0100 Subject: [PATCH 15/18] remove feature completely --- x-pack/plugins/streams/server/plugin.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 2d9f911440b1a..ef070984803d5 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -8,7 +8,6 @@ import { CoreSetup, CoreStart, - DEFAULT_APP_CATEGORIES, KibanaRequest, Logger, Plugin, @@ -59,16 +58,6 @@ export class StreamsPlugin logger: this.logger, } as StreamsServer; - plugins.features.registerKibanaFeature({ - id: 'streams', - name: 'Streams', - order: 1500, - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - category: DEFAULT_APP_CATEGORIES.observability, - privileges: null, - }); - registerRoutes({ repository: StreamsRouteRepository, dependencies: { From 76ed81d675d60daf7aa510d572bf21f0fa2e5b78 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:59:55 +0100 Subject: [PATCH 16/18] cleanup --- x-pack/plugins/streams/kibana.jsonc | 1 - x-pack/plugins/streams/server/types.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index d97a0b371bb27..8e428c0a285a1 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -17,7 +17,6 @@ "usageCollection", "licensing", "taskManager", - "features" ], "optionalPlugins": [ "cloud", diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts index f1cfcb08a2649..f119faa0ed010 100644 --- a/x-pack/plugins/streams/server/types.ts +++ b/x-pack/plugins/streams/server/types.ts @@ -16,7 +16,6 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; -import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import { StreamsConfig } from '../common/config'; export interface StreamsServer { @@ -36,7 +35,6 @@ export interface ElasticsearchAccessorOptions { export interface StreamsPluginSetupDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; taskManager: TaskManagerSetupContract; - features: FeaturesPluginSetup; } export interface StreamsPluginStartDependencies { From 2427d2e5089c478d25dd8c9aa7dabb6520b9eeb2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:11:29 +0000 Subject: [PATCH 17/18] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/streams/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index e0ceaf8197fa6..c2fde35f9ca22 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -27,6 +27,5 @@ "@kbn/zod", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/features-plugin" ] } From a60a16523c07286f78c05137b0d4898d9a050988 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 12 Nov 2024 14:14:08 -0700 Subject: [PATCH 18/18] Updating the owners and CODEOWNERS to @simianhacker @flash1293 @dgieselaar since we don't have an official team for this plugin yet. --- .github/CODEOWNERS | 2 +- x-pack/plugins/streams/kibana.jsonc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ecdb994e3d140..91eee74db6b03 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -966,7 +966,7 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops -x-pack/plugins/streams @elastic/obs-entities +x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index 8e428c0a285a1..06c37ed245cf1 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/streams-plugin", - "owner": "@elastic/obs-entities", + "owner": "@simianhacker @flash1293 @dgieselaar", "description": "A manager for Streams", "group": "observability", "visibility": "private", @@ -16,7 +16,7 @@ "encryptedSavedObjects", "usageCollection", "licensing", - "taskManager", + "taskManager" ], "optionalPlugins": [ "cloud",