From 86260fce046efa45a7a6c5ffe34d3920268724fc Mon Sep 17 00:00:00 2001 From: Mykola Harmash Date: Mon, 7 Apr 2025 15:57:33 +0200 Subject: [PATCH 1/2] [Oblt Onboarding][Auto Detect] Filter out httpjson inputs and fix accidental config backup file (#216978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/elastic/kibana/issues/199744 * Adds a separate `sed` commands for Linux and macOS when replacing API key within the Agent config. GNU and BSD versions of `sed` treat `-i` (in-place editing) argument differently, GNU version allows `-i` without a value while BSD version requires a backup file extension even when it's empty 🫠 * Adds filtering of unsupported input types inside the integration policies. For now it only filters out `httpjson`. ## How to test 1. Go through the auto-detect flow 2. Make sure there is no `'elastic-agent.yml='` file in the Agent directory, or any other weird artifacts 3. Inspect individual integration config files, make sure they don't have `httpjson` inputs Co-authored-by: Joe Reuter (cherry picked from commit ec72d4a880cc7c396fc0376bb33bdb2821bc6db5) # Conflicts: # x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts --- .../public/assets/auto_detect.sh | 9 +++++-- .../server/routes/flow/route.ts | 26 ++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/assets/auto_detect.sh b/x-pack/solutions/observability/plugins/observability_onboarding/public/assets/auto_detect.sh index 009b5821606a3..001611d771c53 100755 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/assets/auto_detect.sh +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/assets/auto_detect.sh @@ -310,9 +310,14 @@ apply_elastic_agent_config() { # Remove existing config file including `inputs.d` directory rm -rf "$elastic_agent_config_path" "$(dirname "$elastic_agent_config_path")/inputs.d" && # Extract new config files from downloaded archive - tar --extract --file "$elastic_agent_tmp_config_path" --directory "$(dirname "$elastic_agent_config_path")" && + tar --extract --file "$elastic_agent_tmp_config_path" --directory "$(dirname "$elastic_agent_config_path")" # Replace placeholder with the Ingest API key - sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" + if [ "${OS}" == "Linux" ]; then + sed -i "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" + else + # macOS requires an empty string for the backup extension + sed -i '' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" + fi if [ "$?" -eq 0 ]; then printf "\e[32;1m✓\e[0m %s\n" "Config files written to:" while IFS= read -r file; do diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts index d67fc15de0fce..49b2e44cec3a2 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts @@ -12,7 +12,7 @@ import { FleetUnauthorizedError, type PackageClient, } from '@kbn/fleet-plugin/server'; -import { safeDump } from 'js-yaml'; +import { load, dump } from 'js-yaml'; import { PackageDataStreamTypes, Output } from '@kbn/fleet-plugin/common/types'; import { transformOutputToFullPolicyOutput } from '@kbn/fleet-plugin/server/services/output_client'; import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../common/telemetry_events'; @@ -454,7 +454,10 @@ async function ensureInstalledIntegrations( if (installSource === 'registry') { const installation = await packageClient.ensureInstalledPackage({ pkgName }); const pkg = installation.package; - const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version); + const config = filterUnsupportedInputs( + await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version) + ); + const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version); return { @@ -479,7 +482,7 @@ async function ensureInstalledIntegrations( pkgName, pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0` title: pkgName, - config: safeDump({ + config: dump({ inputs: [ { id: `filestream-${pkgName}`, @@ -525,6 +528,21 @@ async function ensureInstalledIntegrations( ); } +function filterUnsupportedInputs(policyYML: string): string { + const policy = load(policyYML); + + if (!policy) { + return policyYML; + } + + return dump({ + ...policy, + inputs: (policy.inputs || []).filter((input: any) => { + return input.type !== 'httpjson'; + }), + }); +} + /** * Parses and validates a TSV (tab-separated values) string of integrations with params. * @@ -605,7 +623,7 @@ function generateAgentConfigTar(output: Output, installedIntegrations: Installed path: 'elastic-agent.yml', mode: 0o644, mtime: now, - data: safeDump({ + data: dump({ outputs: { default: transformOutputToFullPolicyOutput(output, undefined, true), }, From 8de1f0c775bd114f7f4ab3351c0558176e95387e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 8 Apr 2025 12:14:18 +0000 Subject: [PATCH 2/2] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/routes/flow/route.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts index 49b2e44cec3a2..38f4405bb7807 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/flow/route.ts @@ -12,7 +12,7 @@ import { FleetUnauthorizedError, type PackageClient, } from '@kbn/fleet-plugin/server'; -import { load, dump } from 'js-yaml'; +import { safeLoad, safeDump } from 'js-yaml'; import { PackageDataStreamTypes, Output } from '@kbn/fleet-plugin/common/types'; import { transformOutputToFullPolicyOutput } from '@kbn/fleet-plugin/server/services/output_client'; import { OBSERVABILITY_ONBOARDING_TELEMETRY_EVENT } from '../../../common/telemetry_events'; @@ -482,7 +482,7 @@ async function ensureInstalledIntegrations( pkgName, pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0` title: pkgName, - config: dump({ + config: safeDump({ inputs: [ { id: `filestream-${pkgName}`, @@ -529,13 +529,13 @@ async function ensureInstalledIntegrations( } function filterUnsupportedInputs(policyYML: string): string { - const policy = load(policyYML); + const policy = safeLoad(policyYML); if (!policy) { return policyYML; } - return dump({ + return safeDump({ ...policy, inputs: (policy.inputs || []).filter((input: any) => { return input.type !== 'httpjson'; @@ -623,7 +623,7 @@ function generateAgentConfigTar(output: Output, installedIntegrations: Installed path: 'elastic-agent.yml', mode: 0o644, mtime: now, - data: dump({ + data: safeDump({ outputs: { default: transformOutputToFullPolicyOutput(output, undefined, true), },