diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 368d3ad3fa8aa..193af296d4256 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -693,6 +693,7 @@ src/platform/packages/shared/kbn-workflows @elastic/workflows-eng src/platform/packages/shared/kbn-workflows-ui @elastic/workflows-eng src/platform/packages/shared/kbn-workspaces @elastic/observability-ui src/platform/packages/shared/kbn-xstate-utils @elastic/obs-onboarding-team +src/platform/packages/shared/kbn-yaml-loader @elastic/fleet src/platform/packages/shared/kbn-zod @elastic/kibana-core src/platform/packages/shared/kbn-zod-helpers @elastic/security-detection-rule-management src/platform/packages/shared/presentation/presentation_publishing @elastic/kibana-presentation diff --git a/package.json b/package.json index b1905e6717bdf..ec7b52a07a27f 100644 --- a/package.json +++ b/package.json @@ -1243,6 +1243,7 @@ "@kbn/workflows-ui": "link:src/platform/packages/shared/kbn-workflows-ui", "@kbn/workplace-ai-app": "link:x-pack/solutions/workplaceai/plugins/workplace_ai_app", "@kbn/xstate-utils": "link:src/platform/packages/shared/kbn-xstate-utils", + "@kbn/yaml-loader": "link:src/platform/packages/shared/kbn-yaml-loader", "@kbn/yaml-rule-editor": "link:x-pack/platform/packages/shared/response-ops/yaml-rule-editor", "@kbn/zod": "link:src/platform/packages/shared/kbn-zod", "@kbn/zod-helpers": "link:src/platform/packages/shared/kbn-zod-helpers", diff --git a/src/platform/packages/shared/kbn-scout/src/playwright/ui_components/monaco_editor.ts b/src/platform/packages/shared/kbn-scout/src/playwright/ui_components/monaco_editor.ts index 8b01e30ca4c45..c70b916fc88ec 100644 --- a/src/platform/packages/shared/kbn-scout/src/playwright/ui_components/monaco_editor.ts +++ b/src/platform/packages/shared/kbn-scout/src/playwright/ui_components/monaco_editor.ts @@ -8,8 +8,8 @@ */ import type { Locator } from '@playwright/test'; -import { expect } from '@playwright/test'; import type { ScoutPage } from '..'; +import { expect } from '../../../ui'; /** * Page object that wraps common interactions with the Kibana Monaco-based code editor. @@ -40,28 +40,34 @@ export class KibanaCodeEditorWrapper { * an empty string is returned. */ async getCodeEditorValue(nthIndex: number = 0): Promise { - return await this.page.evaluate((index) => { - const monacoEnv = (window as any).MonacoEnvironment; + let result = ''; - if (!monacoEnv?.monaco?.editor) { - throw new Error('MonacoEnvironment.monaco.editor is not available'); - } + await expect(async () => { + result = await this.page.evaluate((index) => { + const monacoEnv = (window as any).MonacoEnvironment; + + if (!monacoEnv?.monaco?.editor) { + throw new Error('MonacoEnvironment.monaco.editor is not available'); + } + + const values: string[] = monacoEnv.monaco.editor + .getModels() + .map((model: any) => model.getValue() as string); - const values: string[] = monacoEnv.monaco.editor - .getModels() - .map((model: any) => model.getValue() as string); + if (!values.length) { + return ''; + } - if (!values.length) { - return ''; - } + if (index >= 0 && index < values.length) { + return values[index]!; + } - if (index >= 0 && index < values.length) { - return values[index]!; - } + // Fallback to the first model value if the requested index is out of range + return values[0]!; + }, nthIndex); + }).toPass({ timeout: 30_000 }); - // Fallback to the first model value if the requested index is out of range - return values[0]!; - }, nthIndex); + return result; } /** diff --git a/src/platform/packages/shared/kbn-yaml-loader/index.ts b/src/platform/packages/shared/kbn-yaml-loader/index.ts new file mode 100644 index 0000000000000..c3acda89170a2 --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { loadYaml } from './src/load_yaml'; +export type { Document, Pair } from './src/types'; diff --git a/src/platform/packages/shared/kbn-yaml-loader/kibana.jsonc b/src/platform/packages/shared/kbn-yaml-loader/kibana.jsonc new file mode 100644 index 0000000000000..474103e0f67dd --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/yaml-loader", + "owner": ["@elastic/fleet"], + "group": "platform", + "visibility": "shared", + "description": "Async loader for the yaml package to enable code-splitting and reduce initial bundle size" +} diff --git a/src/platform/packages/shared/kbn-yaml-loader/moon.yml b/src/platform/packages/shared/kbn-yaml-loader/moon.yml new file mode 100644 index 0000000000000..66e7c0aafb46f --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/moon.yml @@ -0,0 +1,32 @@ +# This file is generated by the @kbn/moon package. Any manual edits will be erased! +# To extend this, write your extensions/overrides to 'moon.extend.yml' +# then regenerate this file with: 'node scripts/regenerate_moon_projects.js --update --filter @kbn/yaml-loader' + +$schema: https://moonrepo.dev/schemas/project.json +id: '@kbn/yaml-loader' +layer: unknown +owners: + defaultOwner: '@elastic/fleet' +toolchains: + default: node + javascript: + rootPackageDependenciesOnly: false +language: typescript +project: + title: '@kbn/yaml-loader' + description: Moon project for @kbn/yaml-loader + channel: '' + owner: '@elastic/fleet' + sourceRoot: src/platform/packages/shared/kbn-yaml-loader +dependsOn: [] +tags: + - shared-common + - package + - prod + - group-platform + - shared +fileGroups: + src: + - '**/*.ts' + - '!target/**/*' +tasks: {} diff --git a/src/platform/packages/shared/kbn-yaml-loader/package.json b/src/platform/packages/shared/kbn-yaml-loader/package.json new file mode 100644 index 0000000000000..418139ba4f984 --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/package.json @@ -0,0 +1,11 @@ +{ + "name": "@kbn/yaml-loader", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0", + "main": "index.ts", + "types": "index.ts", + "dependencies": { + "yaml": "2.8.1" + } +} diff --git a/src/platform/packages/shared/kbn-yaml-loader/src/load_yaml.ts b/src/platform/packages/shared/kbn-yaml-loader/src/load_yaml.ts new file mode 100644 index 0000000000000..62d6106f3952c --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/src/load_yaml.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/** + * Loads the yaml package asynchronously. Use this in browser code to avoid + * pulling the full yaml library into the initial bundle. + * The returned promise resolves to the yaml module (parse, stringify, Document, etc.). + */ +export const loadYaml = (): Promise => { + return import('yaml'); +}; diff --git a/src/platform/packages/shared/kbn-yaml-loader/src/types.ts b/src/platform/packages/shared/kbn-yaml-loader/src/types.ts new file mode 100644 index 0000000000000..e1aa918ea4643 --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/src/types.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/** + * Type-only re-exports from the yaml package so consumers can type their APIs + * without pulling in the runtime. Use loadYaml() for runtime usage. + */ +export type { Document, Pair } from 'yaml'; diff --git a/src/platform/packages/shared/kbn-yaml-loader/tsconfig.json b/src/platform/packages/shared/kbn-yaml-loader/tsconfig.json new file mode 100644 index 0000000000000..26c495d191e88 --- /dev/null +++ b/src/platform/packages/shared/kbn-yaml-loader/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@kbn/tsconfig-base/tsconfig.json", + "compilerOptions": { + "outDir": "target/types", + "types": ["node"] + }, + "include": ["**/*.ts"], + "exclude": ["target/**/*"], + "kbn_references": [] +} diff --git a/tsconfig.base.json b/tsconfig.base.json index e9789fa8f8ad4..1874ef1828a60 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -2642,6 +2642,8 @@ "@kbn/workspaces/*": ["src/platform/packages/shared/kbn-workspaces/*"], "@kbn/xstate-utils": ["src/platform/packages/shared/kbn-xstate-utils"], "@kbn/xstate-utils/*": ["src/platform/packages/shared/kbn-xstate-utils/*"], + "@kbn/yaml-loader": ["src/platform/packages/shared/kbn-yaml-loader"], + "@kbn/yaml-loader/*": ["src/platform/packages/shared/kbn-yaml-loader/*"], "@kbn/yaml-rule-editor": ["x-pack/platform/packages/shared/response-ops/yaml-rule-editor"], "@kbn/yaml-rule-editor/*": ["x-pack/platform/packages/shared/response-ops/yaml-rule-editor/*"], "@kbn/yarn-install-scripts": ["packages/kbn-yarn-install-scripts"], diff --git a/x-pack/platform/plugins/shared/fleet/common/services/agent_cm_to_yaml.ts b/x-pack/platform/plugins/shared/fleet/common/services/agent_cm_to_yaml.ts index 44d4a7b772082..173a977d3c5ca 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/agent_cm_to_yaml.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/agent_cm_to_yaml.ts @@ -5,30 +5,14 @@ * 2.0. */ -import type { dump } from 'js-yaml'; - import type { FullAgentConfigMap } from '../types/models/agent_cm'; -const CM_KEYS_ORDER = ['apiVersion', 'kind', 'metadata', 'data']; - -export const fullAgentConfigMapToYaml = ( - policy: FullAgentConfigMap, - toYaml: typeof dump -): string => { - return toYaml(policy, { - skipInvalid: true, - sortKeys: (keyA: string, keyB: string) => { - const indexA = CM_KEYS_ORDER.indexOf(keyA); - const indexB = CM_KEYS_ORDER.indexOf(keyB); - if (indexA >= 0 && indexB < 0) { - return -1; - } +import type { YamlModule } from './yaml_utils'; +import { createYamlKeysSorter, toYaml } from './yaml_utils'; - if (indexA < 0 && indexB >= 0) { - return 1; - } +const CM_KEYS_ORDER = ['apiVersion', 'kind', 'metadata', 'data']; - return indexA - indexB; - }, - }); +export const fullAgentConfigMapToYaml = (policy: FullAgentConfigMap, yaml: YamlModule): string => { + const sortCmKeys = createYamlKeysSorter(CM_KEYS_ORDER, yaml); + return toYaml(policy, { sortMapEntries: sortCmKeys, strict: false }, yaml); }; diff --git a/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.test.ts b/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.test.ts index 58743ea5f6320..fb1acd0c7532a 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.test.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.test.ts @@ -9,6 +9,20 @@ import type { FullAgentPolicy } from '../types'; import { fullAgentPolicyToYaml } from './full_agent_policy_to_yaml'; +// Mock yaml module for testing (matches YamlModule shape) +const mockYaml = { + Document: class { + private data: unknown; + constructor(data: unknown) { + this.data = data; + } + toString() { + return JSON.stringify(this.data); + } + }, + isScalar: () => true, +}; + describe('fullAgentPolicyToYaml', () => { it('should replace secrets', () => { const agentPolicyWithSecrets = { @@ -45,9 +59,9 @@ describe('fullAgentPolicyToYaml', () => { fleet: {}, } as unknown as FullAgentPolicy; - const yaml = fullAgentPolicyToYaml(agentPolicyWithSecrets, (policy) => JSON.stringify(policy)); + const result = fullAgentPolicyToYaml(agentPolicyWithSecrets, mockYaml); - expect(yaml).toMatchInlineSnapshot( + expect(result).toMatchInlineSnapshot( `"{\\"id\\":\\"1234\\",\\"outputs\\":{\\"default\\":{\\"type\\":\\"elasticsearch\\",\\"hosts\\":[\\"http://localhost:9200\\"]}},\\"inputs\\":[{\\"id\\":\\"test_input-secrets-abcd1234\\",\\"revision\\":1,\\"name\\":\\"secrets-1\\",\\"type\\":\\"test_input\\",\\"data_stream\\":{\\"namespace\\":\\"default\\"},\\"use_output\\":\\"default\\",\\"package_policy_id\\":\\"abcd1234\\",\\"package_var_secret\\":\\"\${SECRET_0}\\",\\"input_var_secret\\":\\"\${SECRET_1}\\",\\"streams\\":[{\\"id\\":\\"test_input-secrets.log-abcd1234\\",\\"data_stream\\":{\\"type\\":\\"logs\\",\\"dataset\\":\\"secrets.log\\"},\\"package_var_secret\\":\\"\${SECRET_0}\\",\\"input_var_secret\\":\\"\${SECRET_1}\\",\\"stream_var_secret\\":\\"\${SECRET_2}\\"}],\\"meta\\":{\\"package\\":{\\"name\\":\\"secrets\\",\\"version\\":\\"1.0.0\\"}}}],\\"secret_references\\":[{\\"id\\":\\"secret-id-1\\"},{\\"id\\":\\"secret-id-2\\"},{\\"id\\":\\"secret-id-3\\"}],\\"revision\\":2,\\"agent\\":{},\\"signed\\":{},\\"output_permissions\\":{},\\"fleet\\":{}}"` ); }); diff --git a/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.ts b/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.ts index 7bf5849a1b1be..23c706522b49c 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/full_agent_policy_to_yaml.ts @@ -5,10 +5,11 @@ * 2.0. */ -import type { dump } from 'js-yaml'; - import type { FullAgentPolicy } from '../types'; +import type { YamlModule } from './yaml_utils'; +import { createYamlKeysSorter, toYaml } from './yaml_utils'; + const POLICY_KEYS_ORDER = [ 'id', 'name', @@ -30,34 +31,18 @@ const POLICY_KEYS_ORDER = [ export const fullAgentPolicyToYaml = ( policy: FullAgentPolicy, - toYaml: typeof dump, + yaml: YamlModule, apiKey?: string ): string => { - const yaml = toYaml(policy, { - skipInvalid: true, - sortKeys: _sortYamlKeys, - }); - const formattedYml = apiKey ? replaceApiKey(yaml, apiKey) : yaml; + const sortYamlKeys = createYamlKeysSorter(POLICY_KEYS_ORDER, yaml); + const yamlText = toYaml(policy, { sortMapEntries: sortYamlKeys, strict: false }, yaml); + const formattedYml = apiKey ? replaceApiKey(yamlText, apiKey) : yamlText; if (!policy?.secret_references?.length) return formattedYml; return _formatSecrets(policy.secret_references, formattedYml); }; -export function _sortYamlKeys(keyA: string, keyB: string) { - const indexA = POLICY_KEYS_ORDER.indexOf(keyA); - const indexB = POLICY_KEYS_ORDER.indexOf(keyB); - if (indexA >= 0 && indexB < 0) { - return -1; - } - - if (indexA < 0 && indexB >= 0) { - return 1; - } - - return indexA - indexB; -} - function _formatSecrets( secretRefs: NonNullable, ymlText: string diff --git a/x-pack/platform/plugins/shared/fleet/common/services/index.ts b/x-pack/platform/plugins/shared/fleet/common/services/index.ts index aabd814c731bf..37e8a4d23e20a 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/index.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/index.ts @@ -129,6 +129,8 @@ export { // Cloud Connector accessor module export * from './cloud_connectors'; +export type { YamlModule } from './yaml_utils'; +export { createYamlKeysSorter, toYaml } from './yaml_utils'; export { packageInfoHasOtelInputs, packagePolicyHasOtelInputs, diff --git a/x-pack/platform/plugins/shared/fleet/common/services/output_helpers.test.ts b/x-pack/platform/plugins/shared/fleet/common/services/output_helpers.test.ts index 1086de1a7625d..0985e9bc8747b 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/output_helpers.test.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/output_helpers.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import { getAllowedOutputTypesForAgentPolicy, @@ -93,13 +93,13 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { it('returns true when reserved key is present', () => { const configYml = `queue.mem.events: 1000`; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(true); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(true); }); it('returns false when no reserved key is present', () => { const configYml = `some.random.key: 1000`; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(false); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(false); }); }); @@ -111,7 +111,7 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { events: 1000 `; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(true); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(true); }); it('returns false when no reserved key is present', () => { @@ -121,7 +121,7 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { key: 1000 `; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(false); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(false); }); }); @@ -129,13 +129,13 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { it('returns true when reserved key is present', () => { const configYml = `bulk_max_size`; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(true); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(true); }); it('returns false when no reserved key is present', () => { const configYml = `just a string`; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(false); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(false); }); }); @@ -143,7 +143,7 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { it('returns false when reserved key is present only in a comment', () => { const configYml = `true`; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(false); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(false); }); }); @@ -151,7 +151,7 @@ describe('outputYmlIncludesReservedPerformanceKey', () => { it('returns false when YML is empty', () => { const configYml = ``; - expect(outputYmlIncludesReservedPerformanceKey(configYml, load)).toBe(false); + expect(outputYmlIncludesReservedPerformanceKey(configYml, parse)).toBe(false); }); }); }); diff --git a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.test.ts b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.test.ts index 5b8b84e7ea4f3..d445943c72915 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.test.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import { installationStatuses } from '../constants'; import type { @@ -388,13 +388,13 @@ describe('Fleet - validatePackagePolicy()', () => { }; it('returns no errors for valid package policy', () => { - expect(validatePackagePolicy(validPackagePolicy, mockPackage, load)).toEqual( + expect(validatePackagePolicy(validPackagePolicy, mockPackage, parse)).toEqual( noErrorsValidationResults ); }); it('returns errors for invalid package policy', () => { - expect(validatePackagePolicy(invalidPackagePolicy, mockPackage, load)).toEqual({ + expect(validatePackagePolicy(invalidPackagePolicy, mockPackage, parse)).toEqual({ name: ['Name is required'], description: null, namespace: null, @@ -442,7 +442,7 @@ describe('Fleet - validatePackagePolicy()', () => { enabled: false, })); expect( - validatePackagePolicy({ ...validPackagePolicy, inputs: disabledInputs }, mockPackage, load) + validatePackagePolicy({ ...validPackagePolicy, inputs: disabledInputs }, mockPackage, parse) ).toEqual(noErrorsValidationResults); }); @@ -459,7 +459,7 @@ describe('Fleet - validatePackagePolicy()', () => { validatePackagePolicy( { ...invalidPackagePolicy, inputs: inputsWithDisabledStreams }, mockPackage, - load + parse ) ).toEqual({ name: ['Name is required'], @@ -513,7 +513,7 @@ describe('Fleet - validatePackagePolicy()', () => { ...mockPackage, policy_templates: undefined, }, - load + parse ) ).toEqual({ name: null, @@ -530,7 +530,7 @@ describe('Fleet - validatePackagePolicy()', () => { ...mockPackage, policy_templates: [], }, - load + parse ) ).toEqual({ name: null, @@ -550,7 +550,7 @@ describe('Fleet - validatePackagePolicy()', () => { ...mockPackage, policy_templates: [{} as RegistryPolicyTemplate], }, - load + parse ) ).toEqual({ name: null, @@ -567,7 +567,7 @@ describe('Fleet - validatePackagePolicy()', () => { ...mockPackage, policy_templates: [{ inputs: [] } as unknown as RegistryPolicyTemplate], }, - load + parse ) ).toEqual({ name: null, @@ -605,7 +605,7 @@ describe('Fleet - validatePackagePolicy()', () => { ], }, mockPackage, - load + parse ) ).toEqual({ name: null, @@ -639,7 +639,7 @@ describe('Fleet - validatePackagePolicy()', () => { validatePackagePolicy( INVALID_AWS_POLICY as NewPackagePolicy, AWS_PACKAGE as unknown as PackageInfo, - load + parse ) ).toMatchSnapshot(); }); @@ -650,7 +650,7 @@ describe('Fleet - validatePackagePolicy()', () => { validatePackagePolicy( VALID_AWS_POLICY as NewPackagePolicy, AWS_PACKAGE as unknown as PackageInfo, - load + parse ) ) ).toBe(false); @@ -830,7 +830,7 @@ describe('Fleet - validateConditionalRequiredVars()', () => { const validationResults = validatePackagePolicy( invalidPackagePolicyWithRequiredVars, mockPackageInfoRequireVars, - load + parse ); expect(validationResults).toEqual({ @@ -930,7 +930,7 @@ describe('Fleet - validateConditionalRequiredVars()', () => { const validationResults = validatePackagePolicy( invalidPackagePolicyWithRequiredVars, mockPackageInfoRequireVars, - load + parse ); expect(validationResults).toEqual({ @@ -1034,7 +1034,7 @@ describe('Fleet - validateConditionalRequiredVars()', () => { const validationResults = validatePackagePolicy( invalidPackagePolicyWithRequiredVars, mockPackageInfoRequireVars, - load + parse ); expect(validationResults).toEqual({ @@ -1128,7 +1128,7 @@ describe('Fleet - validateConditionalRequiredVars()', () => { const validationResults = validatePackagePolicy( invalidPackagePolicyWithRequiredVars, mockPackageInfoRequireVars, - load + parse ); expect(validationResults).toEqual({ @@ -1281,7 +1281,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toEqual(['myvariable is required']); @@ -1300,7 +1300,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1318,7 +1318,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1337,7 +1337,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1357,7 +1357,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toEqual(['myvariable is required']); @@ -1376,7 +1376,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toEqual(['myvariable is required']); @@ -1395,7 +1395,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1414,7 +1414,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'integer', }, 'myvariable', - load + parse ); expect(res).toEqual(['Invalid integer']); @@ -1431,7 +1431,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'integer', }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1449,7 +1449,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { multi: true, }, 'myvariable', - load + parse ); expect(res).toEqual(['Invalid integer']); @@ -1467,7 +1467,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { multi: true, }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1490,7 +1490,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { ], }, 'myvariable', - load + parse ); expect(res).toEqual(['Invalid value for select type']); @@ -1511,7 +1511,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { ], }, 'myvariable', - load + parse ); expect(res).toEqual(['Invalid value for select type']); @@ -1532,7 +1532,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { ], }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1553,7 +1553,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { ], }, 'myvariable', - load + parse ); expect(res).toBeNull(); @@ -1569,7 +1569,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { secret: true, }, 'secret_variable', - load + parse ); expect(res).toBeNull(); @@ -1586,7 +1586,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { secret: true, }, 'secret_variable', - load + parse ); expect(res).toBeNull(); @@ -1602,7 +1602,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { secret: true, }, 'secret_variable', - load + parse ); expect(res).toEqual(['Secret reference is invalid, id or ids must be provided']); @@ -1618,7 +1618,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { secret: true, }, 'secret_variable', - load + parse ); expect(res).toEqual(['Secret reference is invalid, id must be a string']); @@ -1635,7 +1635,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { secret: true, }, 'secret_variable', - load + parse ); expect(res).toEqual(['Secret reference is invalid, ids must be an array of strings']); @@ -1654,7 +1654,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'duration', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1671,7 +1671,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'duration', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1688,7 +1688,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'duration', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1705,7 +1705,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'duration', }, 'timeout', - load + parse ); expect(res).toContain('Invalid duration format. Expected format like "1h30m45s"'); @@ -1723,7 +1723,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { min_duration: '1h', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1741,7 +1741,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { min_duration: '1h', }, 'timeout', - load + parse ); expect(res).toContain('Duration is below the minimum allowed value of 1h'); @@ -1759,7 +1759,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { max_duration: '2h', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1777,7 +1777,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { max_duration: '2h', }, 'timeout', - load + parse ); expect(res).toContain('Duration is above the maximum allowed value of 2h'); @@ -1796,7 +1796,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { max_duration: '2h', }, 'timeout', - load + parse ); expect(res).toEqual(null); @@ -1814,7 +1814,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { min_duration: 'invalid', }, 'timeout', - load + parse ); expect(res).toContain('Invalid min_duration specification'); @@ -1832,7 +1832,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { max_duration: 'invalid', }, 'timeout', - load + parse ); expect(res).toContain('Invalid max_duration specification'); @@ -1853,7 +1853,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'data_stream.dataset', - load, + parse, 'input' ); }; @@ -1899,7 +1899,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'data_stream.dataset', - load, + parse, 'input' ); @@ -1917,7 +1917,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'data_stream.dataset', - load, + parse, 'integration' ); @@ -1935,7 +1935,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'test_field', - load, + parse, 'input' ); @@ -1953,7 +1953,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'data_stream.dataset', - load, + parse, 'input' ); @@ -1971,7 +1971,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'text', }, 'data_stream.dataset', - load, + parse, 'input' ); @@ -1991,7 +1991,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2008,7 +2008,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2025,7 +2025,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2042,7 +2042,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2059,7 +2059,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2076,7 +2076,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2093,7 +2093,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2110,7 +2110,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual([expect.stringContaining('Invalid URL format')]); @@ -2128,7 +2128,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { url_allowed_schemes: ['https', 'http'], }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2146,7 +2146,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { url_allowed_schemes: ['https', 'http'], }, 'test_url', - load + parse ); expect(res).toEqual([expect.stringContaining('URL scheme "ftp" is not allowed')]); @@ -2163,7 +2163,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { type: 'url', }, 'test_url', - load + parse ); expect(res).toEqual(null); @@ -2181,7 +2181,7 @@ describe('Fleet - validatePackagePolicyConfig', () => { required: true, }, 'test_url', - load + parse ); expect(res).toEqual([expect.stringContaining('is required')]); @@ -2285,7 +2285,11 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { var_group_selections: { auth_method: 'api_key' }, }; - const result = validatePackagePolicy(policyWithUndefinedApiKey, packageInfoWithVarGroups, load); + const result = validatePackagePolicy( + policyWithUndefinedApiKey, + packageInfoWithVarGroups, + parse + ); // api_key and api_url should have validation errors because they're required by var_group expect(result.vars?.api_key).toEqual([expect.stringContaining('is required')]); @@ -2303,7 +2307,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { var_group_selections: { auth_method: 'api_key' }, }; - const result = validatePackagePolicy(policyWithEmptyApiKey, packageInfoWithVarGroups, load); + const result = validatePackagePolicy(policyWithEmptyApiKey, packageInfoWithVarGroups, parse); // Empty strings are allowed for var_group required vars (same as regular required vars) expect(result.vars?.api_key).toBeNull(); @@ -2323,7 +2327,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { var_group_selections: { auth_method: 'api_key' }, }; - const result = validatePackagePolicy(policyWithApiKeySelected, packageInfoWithVarGroups, load); + const result = validatePackagePolicy(policyWithApiKeySelected, packageInfoWithVarGroups, parse); // api_key and api_url have values, should be valid (null) expect(result.vars?.api_key).toBeNull(); @@ -2346,7 +2350,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { var_group_selections: { auth_method: 'oauth' }, }; - const result = validatePackagePolicy(policyWithOAuthSelected, packageInfoWithVarGroups, load); + const result = validatePackagePolicy(policyWithOAuthSelected, packageInfoWithVarGroups, parse); // api_key and api_url are not in selected option, should be skipped expect(result.vars?.api_key).toBeNull(); @@ -2380,7 +2384,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { const result = validatePackagePolicy( policyWithMissingRequiredVar, packageInfoWithRequiredNonGroupVar, - load + parse ); // required_var is not controlled by var_group but has required: true and undefined value @@ -2412,7 +2416,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { const result = validatePackagePolicy( policyWithEmptyRequiredVar, packageInfoWithRequiredNonGroupVar, - load + parse ); // Empty strings are allowed for regular required vars @@ -2430,7 +2434,7 @@ describe('Fleet - validatePackagePolicy with var_groups', () => { var_group_selections: { auth_method: 'api_key' }, }; - const result = validatePackagePolicy(validPolicy, packageInfoWithVarGroups, load); + const result = validatePackagePolicy(validPolicy, packageInfoWithVarGroups, parse); expect(result.vars?.api_key).toBeNull(); expect(result.vars?.api_url).toBeNull(); diff --git a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts index f4b38f02300aa..045249d158aa1 100644 --- a/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/common/services/validate_package_policy.ts @@ -620,7 +620,9 @@ export const validatePackagePolicyConfig = ( if (varDef.type === 'yaml') { try { - parsedValue = safeLoadYaml(value); + // Coerce to string before parsing to match the behavior of js-yaml.load, + // which internally calls String(input). The yaml package requires a string. + parsedValue = safeLoadYaml(String(value)); } catch (e) { errors.push( i18n.translate('xpack.fleet.packagePolicyValidation.invalidYamlFormatErrorMessage', { diff --git a/x-pack/platform/plugins/shared/fleet/common/services/yaml_utils.ts b/x-pack/platform/plugins/shared/fleet/common/services/yaml_utils.ts new file mode 100644 index 0000000000000..71023f2651aa7 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/common/services/yaml_utils.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Minimal shape of the yaml module required by these utils. + * Callers pass the actual yaml module (from static import or loadYaml()) so that + * common code does not statically import 'yaml' and pull it into the browser bundle. + */ +export interface YamlModule { + Document: new (data: unknown, options?: any) => { toString(): string }; + isScalar: (node: unknown) => boolean; +} + +/** + * Pair-like shape for YAML key sorting (key may be a scalar with .value). + */ +export interface YamlPairLike { + key: { value?: unknown }; +} + +/** + * Creates a YAML key sorter function based on a defined key order. + * Keys in the order array are sorted first, in the specified order. + * Keys not in the array are sorted after, maintaining their relative order. + */ +export function createYamlKeysSorter( + keyOrder: string[], + yaml: YamlModule +): (a: YamlPairLike, b: YamlPairLike) => number { + const { isScalar } = yaml; + return (a, b): number => { + if (!isScalar(a.key) || !isScalar(b.key)) { + return 0; + } + const keyA = a.key.value; + const keyB = b.key.value; + if (typeof keyA !== 'string' || typeof keyB !== 'string') { + return 0; + } + const indexA = keyOrder.indexOf(keyA); + const indexB = keyOrder.indexOf(keyB); + if (indexA >= 0 && indexB < 0) { + return -1; + } + + if (indexA < 0 && indexB >= 0) { + return 1; + } + + return indexA - indexB; + }; +} + +/** + * Converts data to YAML string using the yaml package Document API. + * This is the standard toYaml implementation used across Fleet. + */ +export function toYaml(data: unknown, options: unknown, yaml: YamlModule): string { + const doc = new yaml.Document(data, options); + return doc.toString(); +} diff --git a/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.test.tsx b/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.test.tsx index cd980ea400dc4..f3807b1130493 100644 --- a/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.test.tsx @@ -33,17 +33,15 @@ describe('agent_policy_settings', () => { }); describe('zodStringWithYamlValidation', () => { - it('should accept valid YAML string', () => { - const result = zodStringWithYamlValidation.safeParse( + it('should accept valid YAML string', async () => { + const result = await zodStringWithYamlValidation.safeParseAsync( 'nested:\n key1: value1\n key2: value2' ); expect(result.success).toBe(true); }); - it('should reject invalid YAML string', () => { - const result = zodStringWithYamlValidation.safeParse( - 'nested:\n key1: value1\n key1: value2' - ); + it('should reject invalid YAML string', async () => { + const result = await zodStringWithYamlValidation.safeParseAsync('invalidyaml: [unclosed'); expect(result.success).toBe(false); }); }); diff --git a/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.tsx b/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.tsx index 30c14a7bf5584..b8697ddf7b103 100644 --- a/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.tsx +++ b/x-pack/platform/plugins/shared/fleet/common/settings/agent_policy_settings.tsx @@ -5,13 +5,14 @@ * 2.0. */ import React from 'react'; -import { load } from 'js-yaml'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { z } from '@kbn/zod/v4'; import type { DocLinks } from '@kbn/doc-links'; +import { loadYaml } from '@kbn/yaml-loader'; + import { AGENT_LOG_LEVELS, DEFAULT_LOG_LEVEL } from '../constants'; import type { SettingsConfig } from './types'; @@ -28,11 +29,12 @@ export const zodStringWithDurationValidation = z }); export const zodStringWithYamlValidation = z.string().refine( - (val) => { + async (val) => { + const yaml = await loadYaml(); try { - load(val); + yaml.parse(val); return true; - } catch (error) { + } catch { return false; } }, diff --git a/x-pack/platform/plugins/shared/fleet/cypress/screens/fleet_outputs.ts b/x-pack/platform/plugins/shared/fleet/cypress/screens/fleet_outputs.ts index 5da321c52ba5a..dae53a3082b26 100644 --- a/x-pack/platform/plugins/shared/fleet/cypress/screens/fleet_outputs.ts +++ b/x-pack/platform/plugins/shared/fleet/cypress/screens/fleet_outputs.ts @@ -19,12 +19,21 @@ export const selectESOutput = () => { visit('/app/fleet/settings'); cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('elasticsearch'); + // Wait for the preset input to reach its initial 'balanced' state before + // proceeding. The preset is controlled by a useEffect that depends on the yaml + // module, which loads asynchronously via useYaml(). This assertion ensures the + // component has fully rendered and the async yaml load has been initiated before + // any YAML is typed, so subsequent preset assertions wait only for the yaml + // module to complete loading rather than racing with component initialization. + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', 'balanced'); }; export const selectRemoteESOutput = () => { visit('/app/fleet/settings'); cy.getBySel(SETTINGS_OUTPUTS.ADD_BTN).click(); cy.getBySel(SETTINGS_OUTPUTS.TYPE_INPUT).select('remote_elasticsearch'); + // Same async yaml concern as selectESOutput above. + cy.getBySel(SETTINGS_OUTPUTS.PRESET_INPUT).should('have.value', 'balanced'); }; export const selectKafkaOutput = () => { diff --git a/x-pack/platform/plugins/shared/fleet/cypress/tasks/login.ts b/x-pack/platform/plugins/shared/fleet/cypress/tasks/login.ts index 58c72dd53ae97..f8489a7fb165b 100644 --- a/x-pack/platform/plugins/shared/fleet/cypress/tasks/login.ts +++ b/x-pack/platform/plugins/shared/fleet/cypress/tasks/login.ts @@ -8,7 +8,7 @@ import Url from 'url'; import type { UrlObject } from 'url'; -import * as yaml from 'js-yaml'; +import * as yaml from 'yaml'; import type { ROLES } from './privileges'; import { hostDetailsUrl, LOGOUT_URL } from './navigation'; @@ -245,7 +245,7 @@ const loginViaConfig = () => { // read the login details from `kibana.dev.yaml` cy.readFile(KIBANA_DEV_YML_PATH).then((kibanaDevYml) => { - const config = yaml.load(kibanaDevYml); + const config = yaml.parse(kibanaDevYml); // programmatically authenticate without interacting with the Kibana login page request({ @@ -279,7 +279,7 @@ export const getEnvAuth = (): User => { } else { let user: User = { username: '', password: '' }; cy.readFile(KIBANA_DEV_YML_PATH).then((devYml) => { - const config = yaml.load(devYml); + const config = yaml.parse(devYml); user = { username: config.elasticsearch.username, password: config.elasticsearch.password }; }); diff --git a/x-pack/platform/plugins/shared/fleet/moon.yml b/x-pack/platform/plugins/shared/fleet/moon.yml index a17a78169a10a..f6173112db522 100644 --- a/x-pack/platform/plugins/shared/fleet/moon.yml +++ b/x-pack/platform/plugins/shared/fleet/moon.yml @@ -124,6 +124,7 @@ dependsOn: - '@kbn/rison' - '@kbn/rule-data-utils' - '@kbn/doc-links' + - '@kbn/yaml-loader' tags: - plugin - prod diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/components/agent_policy_yaml_flyout.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/components/agent_policy_yaml_flyout.tsx index a5f2497070130..e7795ea89e03e 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/components/agent_policy_yaml_flyout.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/components/agent_policy_yaml_flyout.tsx @@ -5,10 +5,9 @@ * 2.0. */ -import React, { memo, useCallback } from 'react'; +import React, { memo, useCallback, useEffect, useState } from 'react'; import styled from '@emotion/styled'; import { FormattedMessage } from '@kbn/i18n-react'; -import { dump } from 'js-yaml'; import { EuiCodeBlock, EuiFlexGroup, @@ -29,7 +28,8 @@ import { import { MAX_FLYOUT_WIDTH } from '../constants'; import { useGetOneAgentPolicyFull, useGetOneAgentPolicy, useStartServices } from '../hooks'; -import { fullAgentPolicyToYaml, agentPolicyRouteService } from '../services'; +import { agentPolicyRouteService, getYamlFormatters } from '../services'; +import type { YamlFormatters } from '../../../services/yaml_formatters'; import { API_VERSIONS } from '../../../../common/constants'; import { splitVersionSuffixFromPolicyId } from '../../../../common/services/version_specific_policies_utils'; @@ -48,6 +48,11 @@ export const AgentPolicyYamlFlyout = memo<{ }>(({ policyId, revision, onClose }) => { const flyoutTitleId = useGeneratedHtmlId(); const { version: agentVersion } = splitVersionSuffixFromPolicyId(policyId); + const [formatters, setFormatters] = useState(null); + + useEffect(() => { + getYamlFormatters().then(setFormatters); + }, []); const core = useStartServices(); const { @@ -60,27 +65,28 @@ export const AgentPolicyYamlFlyout = memo<{ (packagePolicy) => packagePolicy?.secret_references?.length ); - const body = isLoadingYaml ? ( - - ) : error ? ( - - } - color="danger" - iconType="warning" - > - {error.message} - - ) : ( - - {fullAgentPolicyToYaml(yamlData!.item, dump)} - - ); + const body = + isLoadingYaml || !formatters ? ( + + ) : error ? ( + + } + color="danger" + iconType="warning" + > + {error.message} + + ) : ( + + {formatters.fullAgentPolicyToYaml(yamlData!.item)} + + ); const revisionQueryParam = revision ? `&revision=${revision}` : ''; const downloadLink = @@ -90,16 +96,16 @@ export const AgentPolicyYamlFlyout = memo<{ const downloadYaml = useCallback( (e: React.MouseEvent) => { e.preventDefault(); - if (!yamlData?.item) { + if (!yamlData?.item || !formatters) { return; } - const yaml = fullAgentPolicyToYaml(yamlData.item, dump); + const yamlStr = formatters.fullAgentPolicyToYaml(yamlData.item); const link = document.createElement('a'); - link.href = `data:text/x-yaml;charset=utf-8,${encodeURIComponent(yaml)}`; + link.href = `data:text/x-yaml;charset=utf-8,${encodeURIComponent(yamlStr)}`; link.download = 'elastic-agent.yml'; link.click(); }, - [yamlData] + [yamlData, formatters] ); return ( @@ -186,7 +192,7 @@ export const AgentPolicyYamlFlyout = memo<{ href={downloadLink} iconType="download" onClick={downloadYaml} - isDisabled={Boolean(isLoadingYaml || !yamlData)} + isDisabled={Boolean(isLoadingYaml || !yamlData || !formatters)} > ) => { - const value = typeName === ZodSchemaType.boolean ? e.target.checked : e.target.value; - const newValue = convertValue(value, typeName); - const validationError = validateSchema(coercedSchema, newValue); - + const applyValidationResult = (validationError: string | undefined) => { if (validationError) { setError(validationError); agentPolicyFormContext?.updateAdvancedSettingsHasErrors(true); @@ -73,6 +69,20 @@ export const SettingsFieldWrapper: React.FC<{ setError(''); agentPolicyFormContext?.updateAdvancedSettingsHasErrors(false); } + }; + + const handleChange = (e: React.ChangeEvent) => { + const value = typeName === ZodSchemaType.boolean ? e.target.checked : e.target.value; + const newValue = convertValue(value, typeName); + + try { + applyValidationResult(validateSchema(coercedSchema, newValue)); + } catch { + // Schema has async refinements (e.g. yaml validation), fall back to async validation + coercedSchema.safeParseAsync(newValue).then((result) => { + applyValidationResult(result.success ? undefined : result.error.issues[0].message); + }); + } const newAdvancedSettings = { ...(agentPolicyFormContext?.agentPolicy.advanced_settings ?? {}), diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.test.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.test.tsx index 156981207e225..9e151f4370f36 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.test.tsx @@ -35,9 +35,12 @@ jest.mock('../../../single_page_layout/hooks/setup_technology', () => { const useAgentlessMock = useAgentless as jest.MockedFunction; +const mockParse = () => ({}); + describe('shouldShowStreamsByDefault', () => { it('should return true if a datastreamId is provided and contained in the input', () => { const res = shouldShowStreamsByDefault( + mockParse, {} as any, [], { @@ -55,6 +58,7 @@ describe('shouldShowStreamsByDefault', () => { it('should return false if a datastreamId is provided but not contained in the input', () => { const res = shouldShowStreamsByDefault( + mockParse, {} as any, [], { @@ -72,6 +76,7 @@ describe('shouldShowStreamsByDefault', () => { it('should return false if a datastreamId is provided but the input is disabled', () => { const res = shouldShowStreamsByDefault( + mockParse, {} as any, [], { diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 72092f3f1122c..d6ed40d819b8a 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState, Fragment, memo, useMemo, useCallback } from 'react'; +import React, { useState, Fragment, memo, useMemo, useCallback, useEffect, useRef } from 'react'; import ReactMarkdown from 'react-markdown'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -34,8 +34,10 @@ import type { RegistryVarsEntry, } from '../../../../../../types'; import type { PackagePolicyInputValidationResults } from '../../../services'; +import type { YamlParseFn } from '../../../services'; import { hasInvalidButRequiredVar, countValidationErrors, isAdvancedVar } from '../../../services'; import { useAgentless } from '../../../single_page_layout/hooks/setup_technology'; +import { useYaml } from '../../../../../../../../services'; import { DATA_STREAM_USE_APM_VAR, shouldIncludeUseAPMVar, @@ -59,6 +61,7 @@ const ShortenedHorizontalRule = styled(EuiHorizontalRule)` `; export const shouldShowStreamsByDefault = ( + parse: YamlParseFn, packageInput: RegistryInput, packageInputStreams: Array, packagePolicyInput: NewPackagePolicyInput, @@ -69,11 +72,12 @@ export const shouldShowStreamsByDefault = ( } return ( - hasInvalidButRequiredVar(packageInput.vars, packagePolicyInput.vars) || + hasInvalidButRequiredVar(parse, packageInput.vars, packagePolicyInput.vars) || packageInputStreams.some( (stream) => stream.enabled && hasInvalidButRequiredVar( + parse, stream.vars, packagePolicyInput.streams.find( (pkgStream) => stream.data_stream.dataset === pkgStream.data_stream.dataset @@ -191,17 +195,35 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ isAgentlessEnabled, onSelectionsChange: (update) => updatePackagePolicyInput(update), }); - // Showing streams toggle state - const [isShowingStreams, setIsShowingStreams] = useState( - () => - (isSingleInputAndStreams && packageInfo.type !== 'input') || - shouldShowStreamsByDefault( - packageInput, - packageInputStreams, - packagePolicyInput, - defaultDataStreamId - ) - ); + const yaml = useYaml(); + // Showing streams toggle state (initialized once when yaml loads so we can validate) + const [isShowingStreams, setIsShowingStreams] = useState(false); + const isShowingStreamsInitialized = useRef(false); + useEffect(() => { + // Only initialize once — we must not re-evaluate after the user fills in required + // fields, or the streams section would collapse as each field becomes valid. + if (yaml && !isShowingStreamsInitialized.current) { + isShowingStreamsInitialized.current = true; + setIsShowingStreams( + (isSingleInputAndStreams && packageInfo.type !== 'input') || + shouldShowStreamsByDefault( + yaml.parse, + packageInput, + packageInputStreams, + packagePolicyInput, + defaultDataStreamId + ) + ); + } + }, [ + yaml, + packageInput, + packageInputStreams, + packagePolicyInput, + defaultDataStreamId, + isSingleInputAndStreams, + packageInfo.type, + ]); // Hide registry variables based on `hide_in_deployment_modes` value const hideRegistryVars = useCallback( diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_configure_package.test.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_configure_package.test.tsx index 6ab7da5ee78a7..38935ef59c5eb 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_configure_package.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_configure_package.test.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { act, fireEvent, waitFor } from '@testing-library/react'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import type { TestRenderer } from '../../../../../../../mock'; import { createFleetTestRendererMock } from '../../../../../../../mock'; @@ -32,7 +32,7 @@ describe('StepConfigurePackage', () => { let testRenderer: TestRenderer; let renderResult: ReturnType; const render = (isAgentlessSelected = false) => { - const validationResults = validatePackagePolicy(packagePolicy, packageInfo, load); + const validationResults = validatePackagePolicy(packagePolicy, packageInfo, parse); renderResult = testRenderer.render( { }); const editPackagePolicy = { ...packagePolicy, supports_agentless: false }; - const validationResults = validatePackagePolicy(editPackagePolicy, packageInfo, load); + const validationResults = validatePackagePolicy(editPackagePolicy, packageInfo, parse); renderResult = testRenderer.render( { ]; const editPackagePolicy = { ...packagePolicy, supports_agentless: false }; - const validationResults = validatePackagePolicy(editPackagePolicy, packageInfo, load); + const validationResults = validatePackagePolicy(editPackagePolicy, packageInfo, parse); renderResult = testRenderer.render( { const validationResults = validatePackagePolicy( singleInputPackagePolicy, singleInputPackageInfo, - load + parse ); renderResult = testRenderer.render( { const validationResults = validatePackagePolicy( singleInputPackagePolicy, singleInputPackageInfo, - load + parse ); renderResult = testRenderer.render( { ], }; - const validationResults = validatePackagePolicy(multiInputPolicy, multiInputPackageInfo, load); + const validationResults = validatePackagePolicy(multiInputPolicy, multiInputPackageInfo, parse); renderResult = testRenderer.render( { const validationResults = validatePackagePolicy( multiTemplatePolicy, multiTemplatePackageInfo, - load + parse ); renderResult = testRenderer.render( { const validationResults = validatePackagePolicy( singleInputPackagePolicy, deprecatedTemplatePackageInfo, - load + parse ); renderResult = testRenderer.render( = ({ children }) => { const [isShowingAdvanced, setIsShowingAdvanced] = useState(false); @@ -90,6 +90,7 @@ export const AddIntegrationPageStep: React.FC = (props props; const { spaceId } = useFleetStatus(); + const yaml = useYaml(); const [basePolicyError, setBasePolicyError] = useState(); const { notifications } = useStartServices(); @@ -106,13 +107,16 @@ export const AddIntegrationPageStep: React.FC = (props inputs: [], }); + const yamlValidationRan = useRef(false); + // Update package policy validation const updatePackagePolicyValidation = useCallback( (newPackagePolicy?: NewPackagePolicy) => { + if (!yaml) return undefined; const newValidationResult = validatePackagePolicy( { ...packagePolicy, ...newPackagePolicy }, packageInfo, - load + yaml.parse ); setValidationResults(newValidationResult); // eslint-disable-next-line no-console @@ -120,7 +124,7 @@ export const AddIntegrationPageStep: React.FC = (props return newValidationResult; }, - [packageInfo, packagePolicy] + [packageInfo, packagePolicy, yaml] ); // Update package policy method const updatePackagePolicy = useCallback( @@ -134,18 +138,36 @@ export const AddIntegrationPageStep: React.FC = (props // eslint-disable-next-line no-console console.debug('Package policy updated', newPackagePolicy); const newValidationResults = updatePackagePolicyValidation(newPackagePolicy); - const hasPackage = newPackagePolicy.package; - const hasValidationErrors = newValidationResults - ? validationHasErrors(newValidationResults) - : false; + if (newValidationResults !== undefined) { + const hasPackage = newPackagePolicy.package; + const hasValidationErrors = validationHasErrors(newValidationResults); + if (hasPackage && !hasValidationErrors) { + setFormState('VALID'); + } else { + setFormState('INVALID'); + } + } + }, + [packagePolicy, updatePackagePolicyValidation] + ); + + // Once yaml loads, run validation against the current package policy so formState + // reflects actual validity rather than the initial optimistic 'VALID' default. + useEffect(() => { + if (yaml && !yamlValidationRan.current) { + yamlValidationRan.current = true; + const newValidationResults = validatePackagePolicy(packagePolicy, packageInfo, yaml.parse); + setValidationResults(newValidationResults); + const hasPackage = packagePolicy.package; + const hasValidationErrors = validationHasErrors(newValidationResults); if (hasPackage && !hasValidationErrors) { setFormState('VALID'); } else { setFormState('INVALID'); } - }, - [packagePolicy, updatePackagePolicyValidation] - ); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [yaml]); // Save package policy const savePackagePolicy = useCallback( diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.test.ts b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.test.ts index 106785191a167..2bc942399141d 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.test.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.test.ts @@ -5,12 +5,15 @@ * 2.0. */ +import { parse } from 'yaml'; + import { hasInvalidButRequiredVar } from './has_invalid_but_required_var'; describe('Fleet - hasInvalidButRequiredVar', () => { it('returns true for invalid & required vars', () => { expect( hasInvalidButRequiredVar( + parse, [ { name: 'mock_var', @@ -24,6 +27,7 @@ describe('Fleet - hasInvalidButRequiredVar', () => { expect( hasInvalidButRequiredVar( + parse, [ { name: 'mock_var', @@ -43,6 +47,7 @@ describe('Fleet - hasInvalidButRequiredVar', () => { it('returns false for valid & required vars', () => { expect( hasInvalidButRequiredVar( + parse, [ { name: 'mock_var', @@ -62,6 +67,7 @@ describe('Fleet - hasInvalidButRequiredVar', () => { it('returns false for optional vars', () => { expect( hasInvalidButRequiredVar( + parse, [ { name: 'mock_var', @@ -78,6 +84,7 @@ describe('Fleet - hasInvalidButRequiredVar', () => { expect( hasInvalidButRequiredVar( + parse, [ { name: 'mock_var', diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.ts b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.ts index e082279a90cd6..a21c037fc3efe 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/has_invalid_but_required_var.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { load } from 'js-yaml'; - import type { PackagePolicyConfigRecord, RegistryVarsEntry, @@ -20,7 +18,10 @@ import { type VarGroupSelection, } from './var_group_helpers'; +export type YamlParseFn = (value: string) => unknown; + export const hasInvalidButRequiredVar = ( + parse: YamlParseFn, registryVars?: RegistryVarsEntry[], packagePolicyVars?: PackagePolicyConfigRecord, varGroups?: RegistryVarGroup[], @@ -81,7 +82,7 @@ export const hasInvalidButRequiredVar = ( packagePolicyVars[registryVar.name], registryVar, registryVar.name, - load, + parse, undefined, requiredByVarGroup ); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts index b2ca8ec47eee6..fa96adcafff5b 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/services/index.ts @@ -6,6 +6,7 @@ */ export { isAdvancedVar } from './is_advanced_var'; +export type { YamlParseFn } from './has_invalid_but_required_var'; export { hasInvalidButRequiredVar } from './has_invalid_but_required_var'; export { getVisibleOptions, diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/form.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/form.tsx index 7bcdfc2a74190..798ae9d9367b7 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/form.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/form.tsx @@ -8,7 +8,6 @@ import React from 'react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { load } from 'js-yaml'; import { isEqual, omit, pick } from 'lodash'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -79,6 +78,7 @@ import { getCloudShellUrlFromPackagePolicy, } from '../../../../../../../components/cloud_security_posture/services'; import { ensurePackageKibanaAssetsInstalled } from '../../../../../services/ensure_kibana_assets_installed'; +import { useYaml } from '../../../../../../../services'; import { useAgentless, useSetupTechnology } from './setup_technology'; @@ -315,6 +315,7 @@ export function useOnSubmit({ }) { const { notifications, docLinks } = useStartServices(); const { spaceId } = useFleetStatus(); + const yaml = useYaml(); const confirmForceInstall = useConfirmForceInstall(); const spaceSettings = useSpaceSettingsContext(); const { canUseMultipleAgentPolicies } = useMultipleAgentPolicies(); @@ -363,11 +364,11 @@ export function useOnSubmit({ // Update package policy validation const updatePackagePolicyValidation = useCallback( (newPackagePolicy?: NewPackagePolicy) => { - if (packageInfo) { + if (packageInfo && yaml) { const newValidationResult = validatePackagePolicy( newPackagePolicy || packagePolicy, packageInfo, - load, + yaml.parse, spaceSettings ); setValidationResults(newValidationResult); @@ -375,7 +376,7 @@ export function useOnSubmit({ return newValidationResult; } }, - [packagePolicy, packageInfo, spaceSettings] + [packagePolicy, packageInfo, spaceSettings, yaml] ); // Update package policy method const updatePackagePolicy = useCallback( diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.test.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.test.tsx index 97bd43bc19f67..fe9d72c2e52cc 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.test.tsx @@ -30,6 +30,10 @@ import { useConfig, } from '../../../../hooks'; +jest.mock('../../../../../../services/use_yaml', () => ({ + useYaml: () => require('yaml'), +})); + jest.mock('../components/steps/components/use_policies', () => { return { ...jest.requireActual('../components/steps/components/use_policies'), diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy.tsx index 3be3908135616..90c4294b06f11 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/use_package_policy.tsx @@ -6,7 +6,6 @@ */ import { useCallback, useEffect, useState } from 'react'; -import { load } from 'js-yaml'; import deepEqual from 'fast-deep-equal'; import { omit, pick } from 'lodash'; @@ -35,6 +34,7 @@ import { validationHasErrors, } from '../../create_package_policy_page/services'; import type { PackagePolicyFormState } from '../../create_package_policy_page/types'; +import { useYaml } from '../../../../../../services'; import { fixApmDurationVars, hasUpgradeAvailable } from '../utils'; import { prepareInputPackagePolicyDataset } from '../../create_package_policy_page/services/prepare_input_pkg_policy_dataset'; @@ -87,6 +87,7 @@ export function usePackagePolicyWithRelatedData( const [loadingError, setLoadingError] = useState(); const [isUpgrade, setIsUpgrade] = useState(options.forceUpgrade ?? false); + const yaml = useYaml(); // Form state const [isEdited, setIsEdited] = useState(false); @@ -116,11 +117,11 @@ export function usePackagePolicyWithRelatedData( // Update package policy validation const updatePackagePolicyValidation = useCallback( (newPackagePolicy?: UpdatePackagePolicy) => { - if (packageInfo) { + if (packageInfo && yaml) { const newValidationResult = validatePackagePolicy( newPackagePolicy || packagePolicy, packageInfo, - load + yaml.parse ); setValidationResults(newValidationResult); // eslint-disable-next-line no-console @@ -129,7 +130,7 @@ export function usePackagePolicyWithRelatedData( return newValidationResult; } }, - [packagePolicy, packageInfo] + [packagePolicy, packageInfo, yaml] ); // Update package policy method const updatePackagePolicy = useCallback( @@ -310,13 +311,13 @@ export function usePackagePolicyWithRelatedData( { prerelease, full: true } ); - if (packageData?.item) { + if (packageData?.item && yaml) { setPackageInfo(packageData.item); const newValidationResults = validatePackagePolicy( newPackagePolicy, packageData.item, - load + yaml.parse ); setValidationResults(newValidationResults); @@ -334,7 +335,14 @@ export function usePackagePolicyWithRelatedData( setIsLoadingData(false); }; getData(); - }, [packagePolicyId, options.forceUpgrade]); + }, [packagePolicyId, options.forceUpgrade, yaml]); + + // Re-run validation when yaml loads (getData may have run before yaml was available) + useEffect(() => { + if (yaml && packageInfo && packagePolicy) { + updatePackagePolicyValidation(); + } + }, [yaml, packageInfo, packagePolicy, updatePackagePolicyValidation]); return { // form diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx index 2eee8b9a7fbfa..a89f4db4c155b 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx @@ -36,6 +36,10 @@ type MockFn = jest.MockedFunction; let lastStepConfigureProps: any; +jest.mock('../../../../../services/use_yaml', () => ({ + useYaml: () => require('yaml'), +})); + jest.mock('../create_package_policy_page/components/steps/components/use_policies', () => { return { ...jest.requireActual('../create_package_policy_page/components/steps/components/use_policies'), diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_effective_config_flyout.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_effective_config_flyout.tsx index c9e0e09ddeb52..bb1f0322c0ae1 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_effective_config_flyout.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_effective_config_flyout.tsx @@ -6,7 +6,6 @@ */ import React, { memo, useMemo } from 'react'; -import { dump } from 'js-yaml'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiButton, @@ -26,9 +25,11 @@ import { import type { Agent } from '../../../../types'; import { MAX_FLYOUT_WIDTH } from '../../../../constants'; import { useGetAgentEffectiveConfigQuery } from '../../../../hooks'; +import { useYaml } from '../../../../../../services'; export const AgentEffectiveConfigFlyout = memo<{ agent: Agent; onClose: () => void }>( ({ agent, onClose }) => { + const yaml = useYaml(); const { data: agentData, isLoading } = useGetAgentEffectiveConfigQuery(agent.id); const agentName = typeof agent.local_metadata?.host?.hostname === 'string' @@ -38,12 +39,12 @@ export const AgentEffectiveConfigFlyout = memo<{ agent: Agent; onClose: () => vo const flyoutTitleId = useGeneratedHtmlId(); const effectiveConfigYaml = useMemo(() => { - if (agentData?.effective_config) { - return dump(agentData.effective_config, { noRefs: true }); + if (agentData?.effective_config && yaml) { + return yaml.stringify(agentData.effective_config); } return ''; - }, [agentData?.effective_config]); + }, [agentData?.effective_config, yaml]); const downloadFile = () => { const link = document.createElement('a'); @@ -68,7 +69,7 @@ export const AgentEffectiveConfigFlyout = memo<{ agent: Agent; onClose: () => vo - {isLoading ? ( + {isLoading || !yaml ? ( ) : ( @@ -87,7 +88,7 @@ export const AgentEffectiveConfigFlyout = memo<{ agent: Agent; onClose: () => vo - + unknown) => (value: string) => { if (value && value !== '') { - const res = load(value); - if ( - typeof res !== 'object' || - Object.values(res).some((val) => { - const valType = typeof val; - return valType !== 'string' && valType !== 'number' && valType !== 'boolean'; - }) - ) { + try { + const res = parse(value); + if ( + typeof res !== 'object' || + res === null || + Object.values(res).some((val) => { + const valType = typeof val; + return valType !== 'string' && valType !== 'number' && valType !== 'boolean'; + }) + ) { + return [ + i18n.translate('xpack.fleet.settings.fleetProxy.proxyHeadersErrorMessage', { + defaultMessage: 'Proxy headers is not a valid key: value object.', + }), + ]; + } + } catch { return [ i18n.translate('xpack.fleet.settings.fleetProxy.proxyHeadersErrorMessage', { defaultMessage: 'Proxy headers is not a valid key: value object.', @@ -72,7 +81,7 @@ function validateProxyHeaders(value: string) { ]; } } -} +}; export function validateName(value: string) { if (!value || value === '') { @@ -86,16 +95,25 @@ export function validateName(value: string) { export function useFleetProxyForm(fleetProxy: FleetProxy | undefined, onSuccess: () => void) { const [isLoading, setIsLoading] = useState(false); + const yaml = useYaml(); const authz = useAuthz(); const { notifications } = useStartServices(); const { confirm } = useConfirmModal(); const isEditDisabled = (!authz.fleet.allSettings || fleetProxy?.is_preconfigured) ?? false; + const validateProxyHeadersFn = yaml ? createValidateProxyHeaders(yaml.parse) : () => undefined; + const proxyHeadersInitialValue = + yaml && fleetProxy?.proxy_headers + ? yaml.stringify(fleetProxy.proxy_headers) + : fleetProxy?.proxy_headers + ? JSON.stringify(fleetProxy.proxy_headers) + : ''; + const nameInput = useInput(fleetProxy?.name ?? '', validateName, isEditDisabled); const urlInput = useInput(fleetProxy?.url ?? '', validateUrl, isEditDisabled); const proxyHeadersInput = useInput( - fleetProxy?.proxy_headers ? dump(fleetProxy.proxy_headers) : '', - validateProxyHeaders, + proxyHeadersInitialValue, + validateProxyHeadersFn, isEditDisabled ); const certificateAuthoritiesInput = useInput( @@ -143,7 +161,8 @@ export function useFleetProxyForm(fleetProxy: FleetProxy | undefined, onSuccess: const data = { name: nameInput.value, url: urlInput.value, - proxy_headers: proxyHeadersInput.value === '' ? undefined : load(proxyHeadersInput.value), + proxy_headers: + proxyHeadersInput.value === '' || !yaml ? undefined : yaml.parse(proxyHeadersInput.value), certificate_authorities: certificateAuthoritiesInput.value, certificate: certificateInput.value, certificate_key: certificateKeyInput.value, @@ -184,6 +203,7 @@ export function useFleetProxyForm(fleetProxy: FleetProxy | undefined, onSuccess: certificateKeyInput.value, validate, notifications, + yaml, confirm, onSuccess, ]); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx index dcb68d93eed7a..4a5d0cd90b8c6 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx @@ -5,9 +5,8 @@ * 2.0. */ -import React, { useMemo, useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; -import { load } from 'js-yaml'; import { EuiFlyout, @@ -50,6 +49,7 @@ import { MAX_FLYOUT_WIDTH } from '../../../../constants'; import type { Output, FleetProxy } from '../../../../types'; import { useBreadcrumbs, useFleetStatus, useStartServices } from '../../../../hooks'; +import { useYaml } from '../../../../../../services'; import { ProxyWarning } from '../fleet_proxies_table/proxy_warning'; @@ -77,8 +77,10 @@ export const EditOutputFlyout: React.FunctionComponent = proxies, }) => { useBreadcrumbs('settings'); + const yaml = useYaml(); const form = useOutputForm(onClose, output, defaultOutput); const inputs = form.inputs; + const parseFn = yaml?.parse; const { docLinks, cloud } = useStartServices(); const fleetStatus = useFleetStatus(); const isServerless = !!cloud?.isServerlessEnabled; @@ -112,6 +114,20 @@ export const EditOutputFlyout: React.FunctionComponent = ? outputTypeSupportPresets(inputs.typeInput.value as ValueOf) : false; + const yamlConfigValue = inputs.additionalYamlConfigInput.value; + const presetValue = inputs.presetInput.value; + const setPresetValue = inputs.presetInput.setValue; + useEffect(() => { + if ( + yaml && + supportsPresets && + outputYmlIncludesReservedPerformanceKey(yamlConfigValue, yaml.parse) && + presetValue !== 'custom' + ) { + setPresetValue('custom'); + } + }, [yaml, supportsPresets, yamlConfigValue, presetValue, setPresetValue]); + const OUTPUT_TYPE_OPTIONS = [ { value: outputType.Elasticsearch, text: 'Elasticsearch' }, { value: outputType.RemoteElasticsearch, text: 'Remote Elasticsearch' }, @@ -448,10 +464,11 @@ export const EditOutputFlyout: React.FunctionComponent = onChange={(e) => inputs.presetInput.setValue(e.target.value)} disabled={ inputs.presetInput.props.disabled || - outputYmlIncludesReservedPerformanceKey( - inputs.additionalYamlConfigInput.value, - load - ) + (!!parseFn && + outputYmlIncludesReservedPerformanceKey( + inputs.additionalYamlConfigInput.value, + parseFn + )) } options={[ { value: 'balanced', text: 'Balanced' }, @@ -505,9 +522,10 @@ export const EditOutputFlyout: React.FunctionComponent = )} {supportsPresets && + !!parseFn && outputYmlIncludesReservedPerformanceKey( inputs.additionalYamlConfigInput.value, - load + parseFn ) && ( <> @@ -558,7 +576,7 @@ export const EditOutputFlyout: React.FunctionComponent = { - if (outputYmlIncludesReservedPerformanceKey(value, load)) { + if (parseFn && outputYmlIncludesReservedPerformanceKey(value, parseFn)) { inputs.presetInput.setValue('custom'); } diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.test.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.test.tsx index 888b815172bc2..29d85cfe57411 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.test.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.test.tsx @@ -5,10 +5,12 @@ * 2.0. */ +import { parse } from 'yaml'; + import { validateESHosts, validateLogstashHosts, - validateYamlConfig, + createValidateYamlConfig, validateCATrustedFingerPrint, validateKafkaHeaders, validateKafkaHosts, @@ -16,6 +18,8 @@ import { validateKibanaAPIKey, } from './output_form_validators'; +const validateYamlConfig = createValidateYamlConfig(parse); + describe('Output form validation', () => { describe('validateKafkaHosts', () => { it('should not work without any urls', () => { diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx index 14d2430f5121a..bcdb634452619 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_validators.tsx @@ -7,7 +7,6 @@ import { i18n } from '@kbn/i18n'; import type { EuiComboBoxOptionOption } from '@elastic/eui'; -import { load } from 'js-yaml'; const toSecretValidator = (validator: (value: string) => string[] | undefined) => @@ -217,19 +216,21 @@ export function validateLogstashHosts(value: string[]) { } } -export function validateYamlConfig(value: string) { +export type YamlParseFn = (value: string) => unknown; + +export const createValidateYamlConfig = (parse: YamlParseFn) => (value: string) => { try { - load(value); + parse(value); return; } catch (error) { return [ i18n.translate('xpack.fleet.settings.outputForm.invalidYamlFormatErrorMessage', { defaultMessage: 'Invalid YAML: {reason}', - values: { reason: error.message }, + values: { reason: (error as Error).message }, }), ]; } -} +}; export function validateName(value: string) { if (!value || value === '') { diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx index 5c5106d9cbdde..10424266dd301 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx @@ -8,10 +8,11 @@ import { useCallback, useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { load } from 'js-yaml'; import type { EuiComboBoxOptionOption } from '@elastic/eui'; +import { useYaml } from '../../../../../../services'; + import { getDefaultPresetForEsOutput } from '../../../../../../../common/services/output_helpers'; import type { @@ -58,7 +59,7 @@ import { validateName, validateESHosts, validateLogstashHosts, - validateYamlConfig, + createValidateYamlConfig, validateCATrustedFingerPrint, validateServiceToken, validateServiceTokenSecret, @@ -198,6 +199,7 @@ export function extractDefaultDynamicKafkaTopics( export function useOutputForm(onSucess: () => void, output?: Output, defaultOutput?: Output) { const fleetStatus = useFleetStatus(); const authz = useAuthz(); + const yaml = useYaml(); const { showExperimentalShipperOptions } = ExperimentalFeaturesService.get(); @@ -231,19 +233,21 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOutp return !allowEdit.includes(field); } + const validateYamlConfigFn = yaml ? createValidateYamlConfig(yaml.parse) : () => undefined; + // Define inputs // Shared inputs const nameInput = useInput(output?.name ?? '', validateName, isDisabled('name')); const typeInput = useInput(output?.type ?? 'elasticsearch', undefined, isDisabled('type')); const additionalYamlConfigInput = useInput( output?.config_yaml ?? '', - validateYamlConfig, + validateYamlConfigFn, isDisabled('config_yaml') ); const otelExporterConfigInput = useInput( (output as NewElasticsearchOutput)?.otel_exporter_config_yaml ?? '', - validateYamlConfig, + validateYamlConfigFn, isDisabled('otel_exporter_config_yaml') ); @@ -276,7 +280,8 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOutp ); const presetInput = useInput( - output?.preset ?? getDefaultPresetForEsOutput(output?.config_yaml ?? '', load), + output?.preset ?? + getDefaultPresetForEsOutput(output?.config_yaml ?? '', yaml?.parse ?? (() => ({}))), () => undefined, isDisabled('preset') ); @@ -333,7 +338,7 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOutp shipper: enabled: false */ - const configJs = output?.config_yaml ? load(output?.config_yaml) : {}; + const configJs = output?.config_yaml && yaml ? yaml.parse(output.config_yaml) : {}; const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; const diskQueueEnabledInput = useSwitchInput(output?.shipper?.disk_queue_enabled ?? false); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/hooks/use_change_log.ts b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/hooks/use_change_log.ts index bbe07cbed99a2..9d300893db70a 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/hooks/use_change_log.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/hooks/use_change_log.ts @@ -7,6 +7,7 @@ import { useMemo } from 'react'; import { useGetFileByPathQuery } from '../../../../../hooks'; +import { useYaml } from '../../../../../../../services'; import { getBreakingChanges, parseYamlChangelog } from '../utils'; /** @@ -19,6 +20,7 @@ export const useChangelog = ( latestVersion: string, currentVersion?: string ) => { + const yaml = useYaml(); const { data, error: getFileError, @@ -28,8 +30,9 @@ export const useChangelog = ( const error = getFileError?.statusCode === 404 ? null : getFileError; const changelog = useMemo(() => { - return parseYamlChangelog(data, latestVersion, currentVersion); - }, [data, latestVersion, currentVersion]); + if (!yaml) return []; + return parseYamlChangelog(yaml.parse, data, latestVersion, currentVersion); + }, [yaml, data, latestVersion, currentVersion]); const breakingChanges = useMemo(() => { const _breakingChanges = getBreakingChanges(changelog); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.test.ts b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.test.ts index 67b4d557c09c1..f38aa6cbd6293 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.test.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.test.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { parse } from 'yaml'; + import { parseYamlChangelog } from '.'; describe('parseYamlChangelog', () => { @@ -31,7 +33,7 @@ describe('parseYamlChangelog', () => { link: https://github.com/elastic/integrations/pull/4399`; it('should return the changelog from latest to current version', () => { - expect(parseYamlChangelog(changelogText, `2.4.0`, `2.2.0`)).toEqual([ + expect(parseYamlChangelog(parse, changelogText, `2.4.0`, `2.2.0`)).toEqual([ { version: '2.4.0', changes: [ @@ -66,7 +68,7 @@ describe('parseYamlChangelog', () => { }); it('should return the changelog to latest version when there is no current version defined', () => { - expect(parseYamlChangelog(changelogText, `2.4.0`)).toEqual([ + expect(parseYamlChangelog(parse, changelogText, `2.4.0`)).toEqual([ { version: '2.4.0', changes: [ @@ -111,9 +113,9 @@ describe('parseYamlChangelog', () => { }); it('should return empty array if changelog text is undefined', () => { - expect(parseYamlChangelog(undefined, `2.4.0`, `2.2.0`)).toEqual([]); + expect(parseYamlChangelog(parse, undefined, `2.4.0`, `2.2.0`)).toEqual([]); }); it('should return empty array if changelog text is null', () => { - expect(parseYamlChangelog(null, `2.4.0`, `2.2.0`)).toEqual([]); + expect(parseYamlChangelog(parse, null, `2.4.0`, `2.2.0`)).toEqual([]); }); }); diff --git a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.ts b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.ts index 6d110870213e2..6a93153920fc6 100644 --- a/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.ts +++ b/x-pack/platform/plugins/shared/fleet/public/applications/integrations/sections/epm/screens/detail/utils/changelog_utils.ts @@ -4,11 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { load } from 'js-yaml'; - import semverGte from 'semver/functions/gte'; import semverLte from 'semver/functions/lte'; +export type YamlParseFn = (value: string) => unknown; + export enum ChangeType { Enhancement = 'enhancement', BreakingChange = 'breaking-change', @@ -40,11 +40,12 @@ export const formatChangelog = (parsedChangelog: Changelog) => { }; export const parseYamlChangelog = ( + parse: YamlParseFn, changelogText: string | null | undefined, latestVersion: string, currentVersion?: string ) => { - const parsedChangelog: Changelog = changelogText ? load(changelogText) : []; + const parsedChangelog: Changelog = changelogText ? (parse(changelogText) as Changelog) : []; if (!currentVersion) return parsedChangelog.filter((e) => semverLte(e.version, latestVersion)); diff --git a/x-pack/platform/plugins/shared/fleet/public/components/agent_enrollment_flyout/hooks.tsx b/x-pack/platform/plugins/shared/fleet/public/components/agent_enrollment_flyout/hooks.tsx index 41035fac70fa7..d5e04c93775d7 100644 --- a/x-pack/platform/plugins/shared/fleet/public/components/agent_enrollment_flyout/hooks.tsx +++ b/x-pack/platform/plugins/shared/fleet/public/components/agent_enrollment_flyout/hooks.tsx @@ -9,8 +9,6 @@ import crypto from 'crypto'; import { useState, useEffect, useMemo, useCallback } from 'react'; import { i18n } from '@kbn/i18n'; -import { dump } from 'js-yaml'; - import type { PackagePolicy, AgentPolicy } from '../../types'; import { sendGetOneAgentPolicy, @@ -37,7 +35,7 @@ import { sendCreateStandaloneAgentAPIKey } from '../../hooks'; import type { FullAgentPolicy } from '../../../common'; -import { fullAgentPolicyToYaml } from '../../services'; +import { getYamlFormatters } from '../../services/yaml_formatters'; import type { K8sMode, @@ -300,7 +298,9 @@ export function useFetchFullPolicy(agentPolicy: AgentPolicy | undefined, isK8s?: if (typeof fullAgentPolicy === 'string') { return; } - setYaml(fullAgentPolicyToYaml(fullAgentPolicy, dump, apiKey)); + getYamlFormatters().then((formatters) => { + setYaml(formatters.fullAgentPolicyToYaml(fullAgentPolicy, apiKey)); + }); } }, [apiKey, fullAgentPolicy, isK8s]); diff --git a/x-pack/platform/plugins/shared/fleet/public/services/index.ts b/x-pack/platform/plugins/shared/fleet/public/services/index.ts index 035e05271a9b0..55ff2f3e85aa1 100644 --- a/x-pack/platform/plugins/shared/fleet/public/services/index.ts +++ b/x-pack/platform/plugins/shared/fleet/public/services/index.ts @@ -28,7 +28,6 @@ export { appRoutesService, packageToPackagePolicy, packageToPackagePolicyInputs, - fullAgentPolicyToYaml, isPackageLimited, doesAgentPolicyAlreadyIncludePackage, isValidNamespace, @@ -50,3 +49,6 @@ export { isPackageUpdatable } from './is_package_updatable'; export { pkgKeyFromPackageInfo } from './pkg_key_from_package_info'; export { createExtensionRegistrationCallback } from './ui_extensions'; export { incrementPolicyName } from './increment_policy_name'; + +export { getYamlFormatters } from './yaml_formatters'; +export { useYaml } from './use_yaml'; diff --git a/x-pack/platform/plugins/shared/fleet/public/services/use_yaml.ts b/x-pack/platform/plugins/shared/fleet/public/services/use_yaml.ts new file mode 100644 index 0000000000000..3a8e79b1e0622 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/public/services/use_yaml.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState, useEffect } from 'react'; + +import { loadYaml } from '@kbn/yaml-loader'; + +type YamlModule = Awaited>; + +let cachedYaml: YamlModule | null = null; +let loadPromise: Promise | null = null; + +/** + * React hook that loads the yaml package asynchronously. + * Returns the yaml module (parse, stringify, Document, etc.) once loaded, or null while loading. + * The module is cached globally so subsequent hook calls resolve synchronously. + */ +export const useYaml = (): YamlModule | null => { + const [yaml, setYaml] = useState(cachedYaml); + + useEffect(() => { + if (cachedYaml) { + setYaml(cachedYaml); + return; + } + + if (!loadPromise) { + loadPromise = loadYaml().catch((err) => { + loadPromise = null; + throw err; + }); + } + + loadPromise.then((mod) => { + cachedYaml = mod; + setYaml(mod); + }); + }, []); + + return yaml; +}; diff --git a/x-pack/platform/plugins/shared/fleet/public/services/yaml_formatters.ts b/x-pack/platform/plugins/shared/fleet/public/services/yaml_formatters.ts new file mode 100644 index 0000000000000..105e6ac7ec1e2 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/public/services/yaml_formatters.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loadYaml } from '@kbn/yaml-loader'; + +import type { FullAgentPolicy } from '../../common'; +import type { FullAgentConfigMap } from '../../common/types/models/agent_cm'; + +import { fullAgentPolicyToYaml } from '../../common/services/full_agent_policy_to_yaml'; +import { fullAgentConfigMapToYaml } from '../../common/services/agent_cm_to_yaml'; + +export interface YamlFormatters { + fullAgentPolicyToYaml: (policy: FullAgentPolicy, apiKey?: string) => string; + fullAgentConfigMapToYaml: (policy: FullAgentConfigMap) => string; +} + +let formattersPromise: Promise | null = null; + +/** + * Returns YAML formatters that use the asynchronously loaded yaml package. + * Result is cached so multiple callers share the same load. + */ +export const getYamlFormatters = (): Promise => { + if (!formattersPromise) { + formattersPromise = loadYaml().then((yaml: Awaited>) => ({ + fullAgentPolicyToYaml: (policy: FullAgentPolicy, apiKey?: string) => + fullAgentPolicyToYaml(policy, yaml, apiKey), + fullAgentConfigMapToYaml: (policy: FullAgentConfigMap) => + fullAgentConfigMapToYaml(policy, yaml), + })); + } + return formattersPromise!; +}; diff --git a/x-pack/platform/plugins/shared/fleet/server/errors/index.ts b/x-pack/platform/plugins/shared/fleet/server/errors/index.ts index a2616e143c0be..a3bdd976fd3ae 100644 --- a/x-pack/platform/plugins/shared/fleet/server/errors/index.ts +++ b/x-pack/platform/plugins/shared/fleet/server/errors/index.ts @@ -18,7 +18,7 @@ export { fleetErrorToResponseOptions, } from './handlers'; -export { isESClientError, rethrowIfInstanceOrWrap } from './utils'; +export { isESClientError, rethrowIfInstanceOrWrap, getErrorMessage } from './utils'; export { FleetError as FleetError, FleetVersionConflictError, diff --git a/x-pack/platform/plugins/shared/fleet/server/errors/utils.ts b/x-pack/platform/plugins/shared/fleet/server/errors/utils.ts index 5683836fc4e89..9b32002b515d0 100644 --- a/x-pack/platform/plugins/shared/fleet/server/errors/utils.ts +++ b/x-pack/platform/plugins/shared/fleet/server/errors/utils.ts @@ -76,6 +76,17 @@ catchAndSetErrorStackTrace.withMessage = (message) => { * rethrowIfInstanceOrWrap(error, CloudConnectorCreateError, 'Failed to create cloud connector'); * } */ +/** + * Extracts a string message from an unknown error value. + * Returns the error's message if it's an Error instance, otherwise converts to string. + * + * @param error - The error value to extract a message from + * @returns The error message as a string + */ +export function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + export function rethrowIfInstanceOrWrap Error>( error: unknown, ErrorClass: T, @@ -87,7 +98,7 @@ export function rethrowIfInstanceOrWrap Error } // Otherwise, wrap it in the target error class - const errorMessage = error instanceof Error ? error.message : String(error); + const errorMessage = getErrorMessage(error); const errorStack = error instanceof Error ? error.stack : undefined; const wrappedMessage = message diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts index f01fe010f1c8f..563e83fc25ff7 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts @@ -8,10 +8,11 @@ import type { TypeOf } from '@kbn/config-schema'; import type { KibanaRequest, RequestHandler, ResponseHeaders } from '@kbn/core/server'; import pMap from 'p-map'; -import { dump } from 'js-yaml'; import { isEmpty, uniq } from 'lodash'; +import yaml from 'yaml'; + import { ALL_SPACES_ID, AGENT_POLICY_VERSION_SEPARATOR, @@ -852,7 +853,7 @@ export const downloadFullAgentPolicy: FleetRequestHandler< }); } const fullAgentPolicy = fleetServerPolicy.data as unknown as FullAgentPolicy; - const body = fullAgentPolicyToYaml(fullAgentPolicy, dump); + const body = fullAgentPolicyToYaml(fullAgentPolicy, yaml); const headers: ResponseHeaders = { 'content-type': 'text/x-yaml', 'content-disposition': `attachment; filename="elastic-agent.yml"`, @@ -894,7 +895,7 @@ export const downloadFullAgentPolicy: FleetRequestHandler< body: { message: 'Agent policy not found' }, }); } - const body = fullAgentPolicyToYaml(fullAgentPolicy, dump); + const body = fullAgentPolicyToYaml(fullAgentPolicy, yaml); const headers: ResponseHeaders = { 'content-type': 'text/x-yaml', 'content-disposition': `attachment; filename="elastic-agent.yml"`, diff --git a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.test.ts b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.test.ts index 39f8b40ffd996..8bd3c2288d4e6 100644 --- a/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/index.test.ts @@ -6,10 +6,11 @@ */ import { httpServerMock } from '@kbn/core-http-server-mocks'; -import { dump } from 'js-yaml'; import { schema } from '@kbn/config-schema'; +import yaml from 'yaml'; + import type { FleetRequestHandlerContext } from '../..'; import { xpackMocks } from '../../mocks'; @@ -27,6 +28,7 @@ import { import { ListResponseSchema } from '../schema/utils'; import { agentPolicyService } from '../../services'; + import { fullAgentPolicyToYaml } from '../../../common/services'; import { @@ -676,7 +678,7 @@ describe('schema validation', () => { }); it('download full agent policy should return valid response', async () => { - const expectedResponse = fullAgentPolicyToYaml(fullAgentPolicy, dump); + const expectedResponse = fullAgentPolicyToYaml(fullAgentPolicy, yaml); (agentPolicyService.getFullAgentPolicy as jest.Mock).mockResolvedValue(fullAgentPolicy); await downloadFullAgentPolicy( context, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts index 5d01ac4abf678..a17a72fe82a76 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts @@ -6,7 +6,7 @@ */ import type { SavedObjectsClientContract } from '@kbn/core/server'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import deepMerge from 'deepmerge'; import { set } from '@kbn/safer-lodash-set'; @@ -534,7 +534,7 @@ export function transformOutputToFullPolicyOutput( preset, } = output; - const configJs = config_yaml ? load(config_yaml) : {}; + const configJs = config_yaml ? parse(config_yaml) : {}; // build logic to read config_yaml and transform it with the new shipper data const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; @@ -686,7 +686,7 @@ export function transformOutputToFullPolicyOutput( } if (outputTypeSupportPresets(output.type)) { - newOutput.preset = preset ?? getDefaultPresetForEsOutput(config_yaml ?? '', load); + newOutput.preset = preset ?? getDefaultPresetForEsOutput(config_yaml ?? '', parse); } return newOutput; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts index 2c4336c850616..dc32cc98e530e 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts @@ -8,7 +8,6 @@ import apm from 'elastic-apm-node'; import { withActiveSpan } from '@kbn/tracing-utils'; import { groupBy, isEqual, keyBy, omit, pick, uniq } from 'lodash'; import { v4 as uuidv4, v5 as uuidv5 } from 'uuid'; -import { dump } from 'js-yaml'; import pMap from 'p-map'; import { lt, minVersion, gt } from 'semver'; import type { @@ -34,6 +33,8 @@ import type { SavedObjectError } from '@kbn/core-saved-objects-common'; import { withSpan } from '@kbn/apm-utils'; +import yaml from 'yaml'; + import { copyPackagePolicy } from '../../common/services/copy_package_policy_utils'; import { catchAndSetErrorStackTrace } from '../errors/utils'; @@ -146,9 +147,9 @@ import { bulkInstallPackages, getPackageInfo } from './epm/packages'; import { ensureInstalledPackage } from './epm/packages/install'; import { getAgentsByKuery, unenrollForAgentPolicyId } from './agents'; import { + getCompiledVersionsForAgentPolicy, getPackagePolicySavedObjectType, packagePolicyService, - getCompiledVersionsForAgentPolicy, } from './package_policy'; import { incrementPackagePolicyCopyName } from './package_policies'; import { outputService } from './output'; @@ -2133,7 +2134,7 @@ class AgentPolicyService { }, }; - const configMapYaml = fullAgentConfigMapToYaml(fullAgentConfigMap, dump); + const configMapYaml = fullAgentConfigMapToYaml(fullAgentConfigMap, yaml); const updateManifestVersion = elasticAgentStandaloneManifest.replace('VERSION', agentVersion); const fixedAgentYML = configMapYaml.replace('agent.yml:', 'agent.yml: |-'); return [fixedAgentYML, updateManifestVersion].join('\n'); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/cloud_connector.ts b/x-pack/platform/plugins/shared/fleet/server/services/cloud_connector.ts index 7a375dcfecac5..09e7023d65941 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/cloud_connector.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/cloud_connector.ts @@ -43,6 +43,7 @@ import { CloudConnectorInvalidVarsError, CloudConnectorDeleteError, rethrowIfInstanceOrWrap, + getErrorMessage, } from '../errors'; import { appContextService } from './app_context'; @@ -269,11 +270,7 @@ export class CloudConnectorService implements CloudConnectorServiceInterface { packagePolicyCount: 0, }; } catch (error) { - logger.error( - `Failed to create cloud connector: ${ - error instanceof Error ? error.message : String(error) - }` - ); + logger.error(`Failed to create cloud connector: ${getErrorMessage(error)}`); rethrowIfInstanceOrWrap( error, CloudConnectorCreateError, @@ -447,11 +444,7 @@ export class CloudConnectorService implements CloudConnectorServiceInterface { packagePolicyCount, }; } catch (error) { - logger.error( - `Failed to update cloud connector: ${ - error instanceof Error ? error.message : String(error) - }` - ); + logger.error(`Failed to update cloud connector: ${getErrorMessage(error)}`); rethrowIfInstanceOrWrap(error, CloudConnectorCreateError, 'Failed to update cloud connector'); } } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/agent/agent.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/agent/agent.ts index 64e88a91769d2..fde5b00883cd0 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/agent/agent.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/agent/agent.ts @@ -6,7 +6,7 @@ */ import Handlebars from '@kbn/handlebars'; -import { load, dump } from 'js-yaml'; +import { parse, stringify } from 'yaml'; import type { Logger } from '@kbn/core/server'; import { coerce, satisfies } from 'semver'; @@ -114,8 +114,21 @@ export function compileTemplate( } compiledTemplate = replaceRootLevelYamlVariables(yamlValues, compiledTemplate); + + // Normalize multi-line double-quoted YAML scalars. The yaml package (unlike + // js-yaml) rejects literal newlines inside double-quoted strings. Values that + // were JSON-stringified and then had parameters resolved may contain actual + // newlines (doubled by handleMultilineStringFormatter). Apply YAML double-quoted + // folding semantics: N consecutive newlines become N-1 \n escape sequences, + // matching the output that js-yaml.load produced from the multi-line format. + compiledTemplate = compiledTemplate.replace(/"(?:[^"\\]|\\.)*"/gs, (match) => + match.includes('\n') + ? match.replace(/\n+/g, (newlines) => '\\n'.repeat(newlines.length - 1)) + : match + ); + try { - const yamlFromCompiledTemplate = load(compiledTemplate, {}); + const yamlFromCompiledTemplate = parse(compiledTemplate); // Hack to keep empty string ('') values around in the end yaml because // `load` replaces empty strings with null @@ -139,21 +152,27 @@ export function compileTemplate( } function handleYamlError(err: any, yaml: string): string { - if (err?.reason === 'duplicated mapping key') { - let position = err.mark?.position; + // Handle duplicated key errors from yaml package + // The yaml package error message format: "Map keys must be unique at line X, column Y:..." + if (err?.code === 'DUPLICATE_KEY') { let key = 'unknown'; - // Read key if position is available - if (position) { - key = ''; - while (position < yaml.length && yaml.charAt(position) !== ':') { - key += yaml.charAt(position); - position++; + // Try to extract the key from the error message or position + if (err.linePos && err.linePos[0]) { + const line = err.linePos[0].line - 1; // Convert to 0-based index + const lines = yaml.split('\n'); + if (line >= 0 && line < lines.length) { + const lineContent = lines[line]; + // Extract key from the line (everything before the colon) + const colonIndex = lineContent.indexOf(':'); + if (colonIndex > 0) { + key = lineContent.substring(0, colonIndex).trim(); + } } } return `YAMLException: Duplicated key "${key}" found in agent policy yaml, please check your yaml variables.`; } - return err.message; + return err?.message || String(err); } function isValidKey(key: string) { @@ -210,7 +229,9 @@ function buildTemplateVariables( if (recordEntry.type && recordEntry.type === 'yaml') { const yamlKeyPlaceholder = `##${key}##`; varPart[lastKeyPart] = recordEntry.value ? `"${yamlKeyPlaceholder}"` : null; - yamlValues[yamlKeyPlaceholder] = recordEntry.value ? load(recordEntry.value) : null; + // Coerce to string before parsing to match the behavior of js-yaml.load, + // which internally calls String(input). The yaml package requires a string. + yamlValues[yamlKeyPlaceholder] = recordEntry.value ? parse(String(recordEntry.value)) : null; } else if (recordEntry.value && recordEntry.value.isSecretRef) { if (recordEntry.value.ids) { varPart[lastKeyPart] = recordEntry.value.ids.map((id: string) => toCompiledSecretRef(id)); @@ -240,24 +261,31 @@ function containsHelper(this: any, item: string, check: string | string[], optio handlebars.registerHelper('contains', containsHelper); // escapeStringHelper will wrap the provided string with single quotes. -// Single quoted strings in yaml need to escape single quotes by doubling them -// and to respect any incoming newline we also need to double them, otherwise -// they will be replaced with a space. +// Single quoted strings in yaml need to escape single quotes by doubling them. +// If the string contains newlines, use double quotes with escaped newlines since the yaml package +// doesn't allow literal newlines in single-quoted strings. function escapeStringHelper(str: string) { if (!str) return undefined; - return "'" + str.replace(/\'/g, "''").replace(/\n/g, '\n\n') + "'"; + if (str.includes('\n')) { + // Use double quotes with escaped newlines for strings with newlines + return '"' + str.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') + '"'; + } + return "'" + str.replace(/\'/g, "''") + "'"; } handlebars.registerHelper('escape_string', escapeStringHelper); /** - * escapeMultilineStringHelper will escape a multiline string by doubling the newlines - * and escaping single quotes. + * escapeMultilineStringHelper will escape a multiline string by escaping single quotes + * and escaping newlines as \n for use inside double quotes. * This is useful when the string is multiline and needs to be escaped in a yaml file * without wrapping it in single quotes. + * Note: Removes spaces immediately following newlines to match js-yaml behavior. */ function escapeMultilineStringHelper(str: string) { if (!str) return undefined; - return str.replace(/\'/g, "''").replace(/\n/g, '\n\n'); + // Remove spaces immediately after newlines to match js-yaml behavior + // then escape single quotes and newlines for use inside double quotes + return str.replace(/\n /g, '\n').replace(/\'/g, "''").replace(/\n/g, '\\n'); } handlebars.registerHelper('escape_multiline_string', escapeMultilineStringHelper); @@ -314,9 +342,15 @@ function replaceRootLevelYamlVariables(yamlVariables: { [k: string]: any }, yaml let patchedTemplate = yamlTemplate; Object.entries(yamlVariables).forEach(([key, val]) => { patchedTemplate = patchedTemplate.replace(new RegExp(`^"${key}"`, 'gm'), () => - val ? dump(val) : '' + val ? stringify(val) : '' ); }); + // Fix flow-style values (e.g. [] or {}) at column 1 that immediately follow a + // mapping key on the preceding line. The yaml package (unlike js-yaml) rejects + // this pattern because the value is not indented under its key. Placing the + // value on the same line as the key resolves the ambiguity. + patchedTemplate = patchedTemplate.replace(/^(\S[^\n]*:)\s*\n(\[.*\]|\{.*\})\s*$/gm, '$1 $2'); + return patchedTemplate; } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.test.ts index 2537eb2fa200d..e98727f172c03 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.test.ts @@ -426,7 +426,7 @@ describe('parseAndVerifyArchive', () => { 'input_only-0.1.0/manifest.yml': buf, }) ).toThrowError( - 'Could not parse top-level package manifest at top-level directory input_only-0.1.0: YAMLException' + 'Could not parse top-level package manifest at top-level directory input_only-0.1.0: Manifest must be a valid YAML object' ); }); @@ -573,7 +573,9 @@ describe('parseAndVerifyDataStreams', () => { 'input-only-0.1.0/data_stream/stream1/manifest.yml': Buffer.alloc(1), }, }) - ).toThrowError("Could not parse package manifest for data stream 'stream1': YAMLException"); + ).toThrowError( + "Could not parse package manifest for data stream 'stream1': Manifest must be a valid YAML object" + ); }); it('should throw when data stream manifest missing type', async () => { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.ts index 733a0141bbeca..2f7c7a2fa7d85 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/archive/parse.ts @@ -11,7 +11,7 @@ import { promisify } from 'util'; import path from 'path'; import { merge } from '@kbn/std'; -import yaml from 'js-yaml'; +import { parse } from 'yaml'; import { pick } from 'lodash'; import semverMajor from 'semver/functions/major'; import semverPrerelease from 'semver/functions/prerelease'; @@ -38,6 +38,7 @@ import { RegistryDataStreamKeys, } from '../../../../common/types'; import { PackageInvalidArchiveError } from '../../../errors'; +import { getErrorMessage } from '../../../errors/utils'; import { pkgToPkgKey } from '../registry'; import { traverseArchiveEntries } from '.'; @@ -240,10 +241,17 @@ export function parseAndVerifyArchive( let manifest: ArchivePackage; try { logger.debug(`Verifying archive - loading yaml`); - manifest = yaml.load(manifestBuffer.toString()); + const parsed = parse(manifestBuffer.toString()); + // Validate that the parsed result is an object (not a primitive like string, number, etc.) + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Manifest must be a valid YAML object'); + } + manifest = parsed; } catch (error) { throw new PackageInvalidArchiveError( - `Could not parse top-level package manifest at top-level directory ${toplevelDir}: ${error}.` + `Could not parse top-level package manifest at top-level directory ${toplevelDir}: ${getErrorMessage( + error + )}.` ); } @@ -320,13 +328,15 @@ export function parseAndVerifyArchive( if (paths.includes(tagsFile) || tagsBuffer) { let tags: PackageSpecTags[]; try { - tags = yaml.load(tagsBuffer.toString()); + tags = parse(tagsBuffer.toString()); logger.debug(`Parsing archive - parsing kibana/tags.yml file`); if (tags.length) { parsed.asset_tags = tags; } } catch (error) { - throw new PackageInvalidArchiveError(`Could not parse tags file kibana/tags.yml: ${error}.`); + throw new PackageInvalidArchiveError( + `Could not parse tags file kibana/tags.yml: ${getErrorMessage(error)}.` + ); } } @@ -378,10 +388,17 @@ export function parseAndVerifyDataStreams(opts: { let manifest; try { - manifest = yaml.load(manifestBuffer.toString()); + const parsed = parse(manifestBuffer.toString()); + // Validate that the parsed result is an object (not a primitive like string, number, etc.) + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Manifest must be a valid YAML object'); + } + manifest = parsed; } catch (error) { throw new PackageInvalidArchiveError( - `Could not parse package manifest for data stream '${dataStreamPath}': ${error}.` + `Could not parse package manifest for data stream '${dataStreamPath}': ${getErrorMessage( + error + )}.` ); } @@ -391,10 +408,12 @@ export function parseAndVerifyDataStreams(opts: { let dataStreamRoutingRules: RegistryDataStreamRoutingRules[] | undefined; if (routingRulesBuffer) { try { - dataStreamRoutingRules = yaml.load(routingRulesBuffer.toString()); + dataStreamRoutingRules = parse(routingRulesBuffer.toString()); } catch (error) { throw new PackageInvalidArchiveError( - `Could not parse routing rules for data stream '${dataStreamPath}': ${error}.` + `Could not parse routing rules for data stream '${dataStreamPath}': ${getErrorMessage( + error + )}.` ); } } @@ -404,10 +423,12 @@ export function parseAndVerifyDataStreams(opts: { let dataStreamLifecyle: RegistryDataStreamLifecycle | undefined; if (lifecyleBuffer) { try { - dataStreamLifecyle = yaml.load(lifecyleBuffer.toString()); + dataStreamLifecyle = parse(lifecyleBuffer.toString()); } catch (error) { throw new PackageInvalidArchiveError( - `Could not parse lifecycle for data stream '${dataStreamPath}': ${error}.` + `Could not parse lifecycle for data stream '${dataStreamPath}': ${getErrorMessage( + error + )}.` ); } } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/esql_views/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/esql_views/install.ts index 3f479a5faf4c6..8dc1fd202a078 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/esql_views/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/esql_views/install.ts @@ -6,7 +6,7 @@ */ import pMap from 'p-map'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; @@ -53,7 +53,7 @@ export async function installEsqlViews({ const esqlViews = esqlViewPaths.map((path) => { const assetData = getAssetFromAssetsMap(esqlViewAssetsMap, path).toString('utf-8'); - const data = path.endsWith('.yml') ? load(assetData) : JSON.parse(assetData); + const data = path.endsWith('.yml') ? parse(assetData) : JSON.parse(assetData); return { name: data.name, query: data.query }; }); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts index 8956f40d261fb..964681d0f2648 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { dump, load } from 'js-yaml'; +import { parse, stringify } from 'yaml'; import { ElasticsearchAssetType } from '../../../../types'; import type { RegistryDataStream } from '../../../../types'; @@ -127,7 +127,7 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor( })); if (pipeline.extension === 'yml') { - const parsedPipelineContent = load(pipeline.contentForInstallation); + const parsedPipelineContent = parse(pipeline.contentForInstallation); customPipelineProcessors.forEach((processor) => mutatePipelineContentWithNewProcessor(parsedPipelineContent, processor) ); @@ -136,7 +136,7 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor( ); return { ...pipeline, - contentForInstallation: `---\n${dump(parsedPipelineContent)}`, + contentForInstallation: `---\n${stringify(parsedPipelineContent, { singleQuote: true })}`, }; } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/meta.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/meta.ts index 71c53efaffd72..aed4a2723a5ef 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/meta.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/meta.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { load, dump } from 'js-yaml'; +import { parse, stringify } from 'yaml'; import type { ESAssetMetadata } from '../../../../common/types'; import { PackagePolicyValidationError } from '../../../../common/errors'; @@ -46,12 +46,12 @@ export function appendMetadataToIngestPipeline({ if (pipeline.extension === 'yml') { // Convert the YML content to JSON, append the `_meta` value, then convert it back to // YML and return the resulting YML - const parsedPipelineContent = load(pipeline.contentForInstallation); + const parsedPipelineContent = parse(pipeline.contentForInstallation); parsedPipelineContent._meta = meta; return { ...pipeline, - contentForInstallation: `---\n${dump(parsedPipelineContent)}`, + contentForInstallation: `---\n${stringify(parsedPipelineContent)}`, }; } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts index 9e3c48e869d95..df37575bd3683 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -8,7 +8,7 @@ import { readFileSync } from 'fs'; import path from 'path'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import { loggerMock } from '@kbn/logging-mocks'; import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; @@ -104,7 +104,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textFieldLiteralYml); + const fields: Field[] = parse(textFieldLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(fieldMapping); @@ -410,7 +410,7 @@ describe('EPM template', () => { it('tests loading base.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = load(fieldsYML); + const fields: Field[] = parse(fieldsYML); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); @@ -421,7 +421,7 @@ describe('EPM template', () => { it('tests loading coredns.logs.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/coredns.logs.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = load(fieldsYML); + const fields: Field[] = parse(fieldsYML); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); @@ -432,7 +432,7 @@ describe('EPM template', () => { it('tests loading system.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/system.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = load(fieldsYML); + const fields: Field[] = parse(fieldsYML); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); @@ -443,7 +443,7 @@ describe('EPM template', () => { it('tests loading cockroachdb_dynamic_templates.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/cockroachdb_dynamic_templates.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); - const fields: Field[] = load(fieldsYML); + const fields: Field[] = parse(fieldsYML); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); @@ -465,7 +465,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(longWithIndexFalseYml); + const fields: Field[] = parse(longWithIndexFalseYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(longWithIndexFalseMapping); @@ -485,7 +485,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithIndexFalseYml); + const fields: Field[] = parse(keywordWithIndexFalseYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithIndexFalseMapping); @@ -505,7 +505,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(textWithStoreTrueYml); + const fields: Field[] = parse(textWithStoreTrueYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(textWithStoreTrueMapping); @@ -537,7 +537,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(textWithMultiFieldsLiteralYml); + const fields: Field[] = parse(textWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(textWithMultiFieldsMapping); @@ -571,7 +571,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithMultiFieldsMapping); @@ -603,7 +603,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithAnalyzedMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithAnalyzedMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping); @@ -634,7 +634,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithNormalizedMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithNormalizedMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping); @@ -663,7 +663,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithMultiFieldsMapping); @@ -692,7 +692,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithMultiFieldsMapping); @@ -713,7 +713,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(dateWithFormatYml); + const fields: Field[] = parse(dateWithFormatYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(dateWithMapping); @@ -747,7 +747,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(keywordWithMultiFieldsLiteralYml); + const fields: Field[] = parse(keywordWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(keywordWithMultiFieldsMapping); @@ -775,7 +775,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(wildcardWithMultiFieldsLiteralYml); + const fields: Field[] = parse(wildcardWithMultiFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(wildcardWithMultiFieldsMapping); @@ -793,7 +793,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldLiteralYml); + const fields: Field[] = parse(objectFieldLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldMapping); @@ -813,7 +813,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldEnabledFalseLiteralYml); + const fields: Field[] = parse(objectFieldEnabledFalseLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldEnabledFalseMapping); @@ -833,7 +833,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldDynamicFalseLiteralYml); + const fields: Field[] = parse(objectFieldDynamicFalseLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldDynamicFalseMapping); @@ -853,7 +853,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldDynamicTrueLiteralYml); + const fields: Field[] = parse(objectFieldDynamicTrueLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldDynamicTrueMapping); @@ -873,7 +873,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldDynamicStrictLiteralYml); + const fields: Field[] = parse(objectFieldDynamicStrictLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldDynamicStrictMapping); @@ -898,7 +898,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyMapping); @@ -925,7 +925,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); @@ -964,7 +964,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); @@ -1003,7 +1003,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); @@ -1042,7 +1042,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); @@ -1081,7 +1081,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const fields: Field[] = parse(objectFieldWithPropertyReversedLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); @@ -1109,7 +1109,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1137,7 +1137,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1172,7 +1172,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1201,7 +1201,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1237,7 +1237,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1265,7 +1265,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(nestedYaml); + const fields: Field[] = parse(nestedYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(expectedMapping); @@ -1283,7 +1283,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(constantKeywordLiteralYaml); + const fields: Field[] = parse(constantKeywordLiteralYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); @@ -1303,7 +1303,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(constantKeywordLiteralYaml); + const fields: Field[] = parse(constantKeywordLiteralYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(constantKeywordMapping)); @@ -1327,7 +1327,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); @@ -1350,7 +1350,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, false); expect(mappings).toEqual(expectedMapping); @@ -1374,7 +1374,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); @@ -1408,7 +1408,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); @@ -1441,7 +1441,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, false); expect(mappings).toEqual(expectedMapping); @@ -1468,7 +1468,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(literalYml); + const fields: Field[] = parse(literalYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(expectedMapping); @@ -1490,7 +1490,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(metaFieldLiteralYaml); + const fields: Field[] = parse(metaFieldLiteralYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(metaFieldMapping)); @@ -1529,7 +1529,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(metaFieldLiteralYaml); + const fields: Field[] = parse(metaFieldLiteralYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(metaFieldMapping)); @@ -1551,7 +1551,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(fieldLiteralYaml); + const fields: Field[] = parse(fieldLiteralYaml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(JSON.stringify(mappings)).toEqual(JSON.stringify(fieldMapping)); @@ -1571,7 +1571,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1596,7 +1596,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1626,7 +1626,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1657,7 +1657,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1689,7 +1689,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1722,7 +1722,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(runtimeFieldMapping); @@ -1754,7 +1754,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(runtimeFieldMapping); @@ -1786,7 +1786,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1820,7 +1820,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(runtimeFieldMapping); @@ -1875,7 +1875,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(runtimeFieldMapping); @@ -1932,7 +1932,7 @@ describe('EPM template', () => { }, ], }; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields, true); expect(mappings).toEqual(runtimeFieldMapping); @@ -1944,7 +1944,7 @@ describe('EPM template', () => { type: object object_type: constant_keyword `; - const fields: Field[] = load(textWithRuntimeFieldsLiteralYml); + const fields: Field[] = parse(textWithRuntimeFieldsLiteralYml); expect(() => { const processedFields = processFields(fields); generateMappings(processedFields); @@ -1965,7 +1965,7 @@ describe('EPM template', () => { }, }, }; - const fields: Field[] = load(flattenedFieldYml); + const fields: Field[] = parse(flattenedFieldYml); const processedFields = processFields(fields); const mappings = generateMappings(processedFields); expect(mappings).toEqual(flattenedFieldMapping); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts index 6f9d8e17a2a1a..9e94c0cc52f85 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/elasticsearch/transform/install.ts @@ -12,7 +12,7 @@ import type { KibanaRequest, } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { uniqBy } from 'lodash'; import pMap from 'p-map'; @@ -210,7 +210,7 @@ const processTransformAssetsPerModule = async ( } const packageAssets = transformsSpecifications.get(transformModuleId); - const content = load(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8')); + const content = parse(getAssetFromAssetsMap(transformAssetsMap, path).toString('utf-8')); // Handling fields.yml and all other files within 'fields' folder if (fileName === TRANSFORM_SPECS_TYPES.FIELDS || isFields(path)) { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.test.ts index 9381403f3f10d..15c526cb69408 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.test.ts @@ -9,7 +9,7 @@ import { readFileSync } from 'fs'; import path from 'path'; import globby from 'globby'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import { getField, processFields, processFieldsWithWildcard } from './field'; import type { Field, Fields } from './field'; @@ -30,7 +30,7 @@ test('tests loading fields.yml', () => { const files = globby.sync(path.join(__dirname, '/tests/*.yml')); for (const file of files) { const fieldsYML = readFileSync(file, 'utf-8'); - const fields: Field[] = load(fieldsYML); + const fields: Field[] = parse(fieldsYML); const processedFields = processFields(fields); // Check that content file and generated file are equal @@ -778,8 +778,8 @@ describe('processFields', () => { Total swap memory. `; - const noWildcardFields: Field[] = load(noWildcardYml); - const wildcardWithObjectTypeFields: Field[] = load(wildcardWithObjectTypeYml); + const noWildcardFields: Field[] = parse(noWildcardYml); + const wildcardWithObjectTypeFields: Field[] = parse(wildcardWithObjectTypeYml); test('Does not add object type when object_type field when is already defined and name has wildcard', () => { expect(processFieldsWithWildcard(wildcardWithObjectTypeFields)).toMatchInlineSnapshot(` diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts index 30482541ff7a1..44efcea47fb58 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/fields/field.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import type { AssetsMap, PackageInstallContext } from '../../../../common/types'; import { getAssetsDataFromAssetsMap } from '../packages/assets'; @@ -323,7 +323,7 @@ export const loadDatastreamsFieldsFromYaml = ( return fieldDefinitionFiles.reduce((acc, file) => { // Make sure it is defined as it is optional. Should never happen. if (file.buffer) { - const tmpFields = load(file.buffer.toString()); + const tmpFields = parse(file.buffer.toString()); // load() returns undefined for empty files, we don't want that if (tmpFields) { acc = acc.concat(tmpFields); @@ -347,7 +347,7 @@ export const loadTransformFieldsFromYaml = ( return fieldDefinitionFiles.reduce((acc, file) => { // Make sure it is defined as it is optional. Should never happen. if (file.buffer) { - const tmpFields = load(file.buffer.toString()); + const tmpFields = parse(file.buffer.toString()); // load() returns undefined for empty files, we don't want that if (tmpFields) { acc = acc.concat(tmpFields); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/ingest_pipeline.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/ingest_pipeline.ts index 3fc05d3bd7b0a..b1333f8adc3e1 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/ingest_pipeline.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/ingest_pipeline.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { dump } from 'js-yaml'; +import { stringify } from 'yaml'; // NOTE: The install methods will take care of adding a reference to a @custom pipeline. We don't need to add one here. export const createDefaultPipeline = (dataset: string, type: string) => { @@ -25,5 +25,5 @@ export const createDefaultPipeline = (dataset: string, type: string) => { managed: true, }, }; - return dump(pipeline); + return stringify(pipeline); }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/manifest.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/manifest.ts index 21dd67cc207ff..936d35c705f13 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/manifest.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/dataset/manifest.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { dump } from 'js-yaml'; +import { stringify } from 'yaml'; import { convertStringToTitle } from '../../utils'; import type { AssetOptions } from '../generate'; @@ -17,5 +17,5 @@ export const createDatasetManifest = (dataset: string, assetOptions: AssetOption title: convertStringToTitle(dataset), type, }; - return dump(manifest); + return stringify(manifest); }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/manifest.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/manifest.ts index ac30f439fecbb..6b1042be5ad10 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/manifest.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/custom_integrations/assets/manifest.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { dump } from 'js-yaml'; +import { stringify } from 'yaml'; import type { AssetOptions } from './generate'; @@ -34,5 +34,5 @@ export const createManifest = (assetOptions: AssetOptions) => { }, }; - return dump(manifest); + return stringify(manifest); }; diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts index 0e611fa9caadc..f894b608bc518 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import pMap from 'p-map'; import type { MMRegExp } from 'minimatch'; import { minimatch } from 'minimatch'; @@ -439,7 +439,7 @@ export async function getInstalledPackageManifests( const parsedManifests = result.saved_objects.reduce>( (acc, asset) => { - acc.set(asset.attributes.asset_path, load(asset.attributes.data_utf8)); + acc.set(asset.attributes.asset_path, parse(asset.attributes.data_utf8)); return acc; }, new Map() diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts index 8854d544f8dd6..fab04416127af 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get_template_inputs.ts @@ -7,10 +7,13 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import { merge } from 'lodash'; -import { dump } from 'js-yaml'; -import yamlDoc from 'yaml'; +import yaml, { type Pair } from 'yaml'; -import { getNormalizedInputs, isIntegrationPolicyTemplate } from '../../../../common/services'; +import { + getNormalizedInputs, + isIntegrationPolicyTemplate, + createYamlKeysSorter, +} from '../../../../common/services'; import { getStreamsForInputType, @@ -28,7 +31,6 @@ import type { PackagePolicyConfigRecordEntry, RegistryInput, } from '../../../../common/types'; -import { _sortYamlKeys } from '../../../../common/services/full_agent_policy_to_yaml'; import { generateOtelcolConfig } from '../../agent_policies/otel_collector'; import { OTEL_COLLECTOR_INPUT_TYPE } from '../../../../common/constants'; import { getInputsWithIds } from '../../package_policies/get_input_with_ids'; @@ -40,6 +42,27 @@ import { getAgentTemplateAssetsMap } from './get'; type Format = 'yml' | 'json'; +const POLICY_KEYS_ORDER = [ + 'id', + 'name', + 'revision', + 'dataset', + 'type', + 'outputs', + 'fleet', + 'output_permissions', + 'agent', + 'inputs', + 'enabled', + 'use_output', + 'meta', + 'input', + 'download', + 'signed', +]; + +const _sortYamlKeys = createYamlKeysSorter(POLICY_KEYS_ORDER, yaml); + type PackageWithInputAndStreamIndexed = Record< string, RegistryInput & { @@ -234,14 +257,13 @@ export async function getTemplateInputs( if (format === 'json') { return { inputs: filteredInputs, ...(otelcolConfig ? otelcolConfig : {}) }; } else if (format === 'yml') { - const yaml = dump( - { inputs: filteredInputs, ...(otelcolConfig ? otelcolConfig : {}) }, - { - skipInvalid: true, - sortKeys: _sortYamlKeys, - } - ); - return addCommentsToYaml(yaml, buildIndexedPackage(packageInfo), inputIdsDestinationMap); + const data = { inputs: filteredInputs, ...(otelcolConfig ? otelcolConfig : {}) }; + const doc = new yaml.Document(data, { + sortMapEntries: _sortYamlKeys as (a: Pair, b: Pair) => number, + strict: false, + }); + const yamlStr = doc.toString({ singleQuote: true }); + return addCommentsToYaml(yamlStr, buildIndexedPackage(packageInfo), inputIdsDestinationMap); } return { inputs: [] }; @@ -306,20 +328,20 @@ function buildIndexedPackage(packageInfo: PackageInfo): PackageWithInputAndStrea } function addCommentsToYaml( - yaml: string, + yamlStr: string, packageIndexInputAndStreams: PackageWithInputAndStreamIndexed, inputIdsDestinationMap: Map }> ) { - const doc = yamlDoc.parseDocument(yaml); + const doc = yaml.parseDocument(yamlStr); // Add input and streams comments const yamlInputs = doc.get('inputs'); - if (yamlDoc.isCollection(yamlInputs)) { + if (yaml.isCollection(yamlInputs)) { yamlInputs.items.forEach((inputItem) => { - if (!yamlDoc.isMap(inputItem)) { + if (!yaml.isMap(inputItem)) { return; } const inputIdNode = inputItem.get('id', true); - if (!yamlDoc.isScalar(inputIdNode)) { + if (!yaml.isScalar(inputIdNode)) { return; } const inputId = @@ -334,15 +356,15 @@ function addCommentsToYaml( commentVariablesInYaml(inputItem, pkgInput.vars ?? []); const yamlStreams = inputItem.get('streams'); - if (!yamlDoc.isCollection(yamlStreams)) { + if (!yaml.isCollection(yamlStreams)) { return; } yamlStreams.items.forEach((streamItem) => { - if (!yamlDoc.isMap(streamItem)) { + if (!yaml.isMap(streamItem)) { return; } const streamIdNode = streamItem.get('id', true); - if (yamlDoc.isScalar(streamIdNode)) { + if (yaml.isScalar(streamIdNode)) { const streamId = inputIdsDestinationMap .get(inputIdNode.value as string) @@ -360,13 +382,13 @@ function addCommentsToYaml( }); } - return doc.toString(); + return doc.toString({ singleQuote: true }); } -function commentVariablesInYaml(rootNode: yamlDoc.Node, vars: RegistryVarsEntry[] = []) { +function commentVariablesInYaml(rootNode: yaml.Node, vars: RegistryVarsEntry[] = []) { // Node need to be deleted after the end of the visit to be able to visit every node const toDeleteFn: Array<() => void> = []; - yamlDoc.visit(rootNode, { + yaml.visit(rootNode, { Scalar(key, node, path) { if (node.value) { const val = node.value.toString(); @@ -377,17 +399,17 @@ function commentVariablesInYaml(rootNode: yamlDoc.Node, vars: RegistryVarsEntry[ const paths = [...path].reverse(); - let prevPart: yamlDoc.Node | yamlDoc.Document | yamlDoc.Pair = node; + let prevPart: yaml.Node | yaml.Document | yaml.Pair = node; for (const pathPart of paths) { - if (yamlDoc.isCollection(pathPart)) { + if (yaml.isCollection(pathPart)) { // If only one items in the collection comment the whole collection if (pathPart.items.length === 1) { continue; } } - if (yamlDoc.isSeq(pathPart)) { - const commentDoc = new yamlDoc.Document(new yamlDoc.YAMLSeq()); + if (yaml.isSeq(pathPart)) { + const commentDoc = new yaml.Document(new yaml.YAMLSeq()); commentDoc.add(prevPart); const commentStr = commentDoc.toString().trimEnd(); pathPart.comment = pathPart.comment @@ -398,16 +420,16 @@ function commentVariablesInYaml(rootNode: yamlDoc.Node, vars: RegistryVarsEntry[ toDeleteFn.push(() => { pathPart.items.forEach((item, index) => { if (item === keyToDelete) { - pathPart.delete(new yamlDoc.Scalar(index)); + pathPart.delete(new yaml.Scalar(index)); } }); }); return; } - if (yamlDoc.isMap(pathPart)) { - if (yamlDoc.isPair(prevPart)) { - const commentDoc = new yamlDoc.Document(new yamlDoc.YAMLMap()); + if (yaml.isMap(pathPart)) { + if (yaml.isPair(prevPart)) { + const commentDoc = new yaml.Document(new yaml.YAMLMap()); commentDoc.add(prevPart); const commentStr = commentDoc.toString().trimEnd(); diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/update_custom_integration.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/update_custom_integration.ts index 84261057c922a..7e696e6a90e61 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/update_custom_integration.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/update_custom_integration.ts @@ -9,7 +9,7 @@ import type { SavedObject, SavedObjectsClientContract, } from '@kbn/core/server'; -import { load, dump } from 'js-yaml'; +import { parse, stringify } from 'yaml'; import { PACKAGES_SAVED_OBJECT_TYPE, @@ -98,10 +98,10 @@ export async function incrementVersionAndUpdate( const assetsMap = [...installedPkg!.assetsMap.entries()].reduce((acc, [path, content]) => { if (path === `${pkgName}-${installedPkg!.installation.install_version}/manifest.yml`) { - const yaml = load(content!.toString()); + const yaml = parse(content!.toString()); yaml.version = data.version; - content = Buffer.from(dump(yaml)); + content = Buffer.from(stringify(yaml)); } acc.set( @@ -116,10 +116,10 @@ export async function incrementVersionAndUpdate( const manifestPath = `${pkgName}-${data.version}/manifest.yml`; const manifest = assetsMap.get(manifestPath); if (manifest) { - const yaml = load(manifest?.toString()); + const yaml = parse(manifest?.toString()); if (yaml) { yaml.categories = data.categories || []; - assetsMap.set(manifestPath, Buffer.from(dump(yaml))); + assetsMap.set(manifestPath, Buffer.from(stringify(yaml))); } } } @@ -133,7 +133,7 @@ export async function incrementVersionAndUpdate( const changelogPath = `${pkgName}-${data.version}/changelog.yml`; const changelog = assetsMap.get(changelogPath); if (changelog) { - const yaml = load(changelog?.toString()); + const yaml = parse(changelog?.toString()); if (yaml) { const newChangelogItem = { version: data.version, @@ -147,7 +147,7 @@ export async function incrementVersionAndUpdate( ], }; yaml.push(newChangelogItem); - assetsMap.set(changelogPath, Buffer.from(dump(yaml))); + assetsMap.set(changelogPath, Buffer.from(stringify(yaml))); } } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.test.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.test.ts index 166687a836fb1..2d5a19cc47564 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.test.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { dump } from 'js-yaml'; +import { stringify } from 'yaml'; import type { AssetsMap } from '../../../../common/types'; @@ -14,7 +14,7 @@ import type { RegistryDataStream } from '../../../../common'; import { resolveDataStreamFields } from './utils'; describe('resolveDataStreamFields', () => { - const statusAssetYml = dump([ + const statusAssetYml = stringify([ { name: 'apache.status', type: 'group', diff --git a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.ts b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.ts index 59a95a72d4afd..127e807c178e4 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/utils.ts @@ -9,7 +9,7 @@ import { withSpan } from '@kbn/apm-utils'; import type { FieldMetadataPlain } from '@kbn/fields-metadata-plugin/common'; import type { ExtractedDatasetFields } from '@kbn/fields-metadata-plugin/server'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import type { RegistryDataStream } from '../../../../common'; import type { AssetsMap } from '../../../../common/types'; @@ -90,7 +90,7 @@ export const resolveDataStreamFields = ({ const fieldsAssetBuffer = assetsMap.get(fieldsAssetPath); if (fieldsAssetBuffer) { - const fieldsAssetJSON = load(fieldsAssetBuffer.toString('utf8')); + const fieldsAssetJSON = parse(fieldsAssetBuffer.toString('utf8')); const normalizedFields = normalizeFields(fieldsAssetJSON); Object.assign(dataStreamFields, normalizedFields); } diff --git a/x-pack/platform/plugins/shared/fleet/server/services/form_settings/form_settings.ts b/x-pack/platform/plugins/shared/fleet/server/services/form_settings/form_settings.ts index 5966a2f2a0499..c5433efc47fb4 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/form_settings/form_settings.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/form_settings/form_settings.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { load } from 'js-yaml'; +import { parse } from 'yaml'; +import { i18n } from '@kbn/i18n'; import { type Props, schema } from '@kbn/config-schema'; import { stringifyZodError } from '@kbn/zod-helpers/v4'; @@ -30,6 +31,19 @@ export function _getSettingsAPISchema(settings: SettingsConfig[]): Props { schema.literal(null), schema.any({ validate: (val: any) => { + if (setting.type === 'yaml') { + try { + parse(val); + } catch { + return i18n.translate( + 'xpack.fleet.settings.agentPolicyAdvanced.yamlValidationMessage', + { + defaultMessage: 'Must be a valid YAML string', + } + ); + } + return; + } const res = setting.schema.safeParse(val); if (!res.success) { return stringifyZodError(res.error); @@ -79,7 +93,7 @@ export function _getSettingsValuesForAgentPolicy( function convertValue(val: any, type?: string) { if (type === 'yaml') { - const valJs = load(val); + const valJs = parse(val); if (valJs.agent?.internal) { return valJs.agent.internal; } else { diff --git a/x-pack/platform/plugins/shared/fleet/server/services/output.ts b/x-pack/platform/plugins/shared/fleet/server/services/output.ts index 1174ac0f6bf01..6ffdd99bcdc54 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/output.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/output.ts @@ -6,7 +6,7 @@ */ import { v5 as uuidv5 } from 'uuid'; import { omit } from 'lodash'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import deepEqual from 'fast-deep-equal'; import { indexBy } from 'lodash/fp'; @@ -557,7 +557,7 @@ class OutputService { if (outputTypeSupportPresets(data.type)) { if ( data.preset === 'balanced' && - outputYmlIncludesReservedPerformanceKey(output.config_yaml ?? '', load) + outputYmlIncludesReservedPerformanceKey(output.config_yaml ?? '', parse) ) { throw new OutputInvalidError( `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( @@ -634,11 +634,11 @@ class OutputService { } if (!data.preset && data.type === outputType.Elasticsearch) { - data.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', load); + data.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', parse); } if (output.config_yaml) { - const configJs = load(output.config_yaml); + const configJs = parse(output.config_yaml); const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; if (isShipperDisabled && output.shipper) { @@ -948,7 +948,7 @@ class OutputService { if (updateData.type && outputTypeSupportPresets(updateData.type)) { if ( updateData.preset === 'balanced' && - outputYmlIncludesReservedPerformanceKey(updateData.config_yaml ?? '', load) + outputYmlIncludesReservedPerformanceKey(updateData.config_yaml ?? '', parse) ) { throw new OutputInvalidError( `preset cannot be balanced when config_yaml contains one of ${RESERVED_CONFIG_YML_KEYS.join( @@ -1142,7 +1142,7 @@ class OutputService { } if (!data.preset && data.type === outputType.Elasticsearch) { - updateData.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', load); + updateData.preset = getDefaultPresetForEsOutput(data.config_yaml ?? '', parse); } // Remove the shipper data if the shipper is not enabled from the yaml config @@ -1150,7 +1150,7 @@ class OutputService { updateData.shipper = null; } if (data.config_yaml) { - const configJs = load(data.config_yaml); + const configJs = parse(data.config_yaml); const isShipperDisabled = !configJs?.shipper || configJs?.shipper?.enabled === false; if (isShipperDisabled && data.shipper) { @@ -1226,7 +1226,7 @@ class OutputService { await pMap( outputs.items.filter((output) => outputTypeSupportPresets(output.type) && !output.preset), async (output) => { - const preset = getDefaultPresetForEsOutput(output.config_yaml ?? '', load); + const preset = getDefaultPresetForEsOutput(output.config_yaml ?? '', parse); await outputService.update( soClient, diff --git a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts index 252130ef96771..0e7ae33362faa 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts @@ -26,7 +26,7 @@ import type { import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; import { v4 as uuidv4 } from 'uuid'; -import { load } from 'js-yaml'; +import { parse } from 'yaml'; import semverGt from 'semver/functions/gt'; import { ALL_SPACES_ID, DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; @@ -3517,7 +3517,7 @@ class PackagePolicyClientWithAuthz extends PackagePolicyClientImpl { } function validatePackagePolicyOrThrow(packagePolicy: NewPackagePolicy, pkgInfo: PackageInfo) { - const validationResults = validatePackagePolicy(packagePolicy, pkgInfo, load); + const validationResults = validatePackagePolicy(packagePolicy, pkgInfo, parse); if (validationHasErrors(validationResults)) { const responseFormattedValidationErrors = Object.entries(getFlattenedObject(validationResults)) .map(([key, value]) => { @@ -4155,7 +4155,7 @@ export function updatePackageInputs( vars, }; - const validationResults = validatePackagePolicy(resultingPackagePolicy, packageInfo, load); + const validationResults = validatePackagePolicy(resultingPackagePolicy, packageInfo, parse); if (validationHasErrors(validationResults)) { const responseFormattedValidationErrors = Object.entries(getFlattenedObject(validationResults)) diff --git a/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts b/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts index 006ca45efe433..a8b3389415d82 100644 --- a/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts +++ b/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts @@ -10,7 +10,7 @@ import utils from 'node:util'; import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { isEqual } from 'lodash'; -import { dump } from 'js-yaml'; +import { stringify } from 'yaml'; import pMap from 'p-map'; const pbkdf2Async = utils.promisify(crypto.pbkdf2); @@ -103,7 +103,7 @@ export async function createOrUpdatePreconfiguredOutputs( const { id, config, ...outputData } = output; - const configYaml = config ? dump(config) : undefined; + const configYaml = config ? stringify(config) : undefined; const data: NewOutput = { ...outputData, diff --git a/x-pack/platform/plugins/shared/fleet/tsconfig.json b/x-pack/platform/plugins/shared/fleet/tsconfig.json index 17b7339f10766..4827e103d6e1b 100644 --- a/x-pack/platform/plugins/shared/fleet/tsconfig.json +++ b/x-pack/platform/plugins/shared/fleet/tsconfig.json @@ -134,6 +134,7 @@ "@kbn/css-utils", "@kbn/rison", "@kbn/rule-data-utils", - "@kbn/doc-links" + "@kbn/doc-links", + "@kbn/yaml-loader" ] } diff --git a/x-pack/platform/test/fleet_api_integration/apis/epm/install_by_upload.ts b/x-pack/platform/test/fleet_api_integration/apis/epm/install_by_upload.ts index 0664f2b8437a0..5ba16449f22ec 100644 --- a/x-pack/platform/test/fleet_api_integration/apis/epm/install_by_upload.ts +++ b/x-pack/platform/test/fleet_api_integration/apis/epm/install_by_upload.ts @@ -251,7 +251,7 @@ export default function (providerContext: FtrProviderContext) { .send(buf) .expect(400); expect((res.error as HTTPError).text).to.equal( - '{"statusCode":400,"error":"Bad Request","message":"Could not parse top-level package manifest at top-level directory apache-0.1.4: YAMLException: bad indentation of a mapping entry (2:7)\\n\\n 1 | format_version: 1.0.0\\n 2 | name: apache\\n-----------^\\n 3 | title: Apache\\n 4 | version: 0.1.4."}' + '{"statusCode":400,"error":"Bad Request","message":"Could not parse top-level package manifest at top-level directory apache-0.1.4: Nested mappings are not allowed in compact mappings at line 1, column 17:\\n\\nformat_version: 1.0.0\\n ^\\n."}' ); }); diff --git a/yarn.lock b/yarn.lock index bc8cad20c2ac3..4670b606cd17c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9827,6 +9827,10 @@ version "0.0.0" uid "" +"@kbn/yaml-loader@link:src/platform/packages/shared/kbn-yaml-loader": + version "0.0.0" + uid "" + "@kbn/yaml-rule-editor@link:x-pack/platform/packages/shared/response-ops/yaml-rule-editor": version "0.0.0" uid ""