[Fleet] Extend OTel exporter configuration

packages/kbn-api-contracts/allowlist.json

@@ -9,6 +9,13 @@
       "approvedBy": "@elastic/terraform-provider",
       "prUrl": "https://github.com/elastic/kibana/pull/258986"
     },
+    {
+      "path": "/api/fleet/outputs/{outputId}",
+      "method": "put",
+      "reason": "PUT request body uses UpdateOutputSchema with meta IDs, changing inline anyOf members to $ref components. Data shape is identical; adds otel_exporter_config_yaml field.",
+      "approvedBy": "elastic/fleet",
+      "prUrl": "https://github.com/elastic/kibana/pull/259308"
+    },
     {
       "path": "/api/fleet/agent_policies",
       "method": "get",

packages/kbn-check-saved-objects-cli/current_fields.json

@@ -859,6 +859,7 @@
     "is_preconfigured",
     "key",
     "name",
+    "otel_exporter_config_yaml",
     "output_id",
     "partition",
     "password",

packages/kbn-check-saved-objects-cli/current_mappings.json

@@ -2849,6 +2849,9 @@
       "name": {
         "type": "keyword"
       },
+      "otel_exporter_config_yaml": {
+        "type": "text"
+      },
       "output_id": {
         "index": false,
         "type": "keyword"

packages/kbn-check-saved-objects-cli/src/migrations/__fixtures__/ingest-outputs/10.9.0.json

@@ -0,0 +1,22 @@
+{
+  "10.8.0": [
+    {
+      "name": "default",
+      "type": "elasticsearch",
+      "is_default": true,
+      "is_default_monitoring": true,
+      "hosts": ["https://localhost:9200"],
+      "preset": "balanced"
+    }
+  ],
+  "10.9.0": [
+    {
+      "name": "default",
+      "type": "elasticsearch",
+      "is_default": true,
+      "is_default_monitoring": true,
+      "hosts": ["https://localhost:9200"],
+      "preset": "balanced"
+    }
+  ]
+}

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -134,7 +134,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "498c2ba7abd4329a0d8b40efd98b4b16991107512d38141707f9f2e10521b367",
         "ingest-agent-policies": "ce61ba4808deecd27dce770799be59f212ebe1c0d76bcc4298aafa4cfce7aa15",
         "ingest-download-sources": "c87e062ef293585e85fccec0c865d7cef48e0ff9a919d7781d5f7627d275484b",
-        "ingest-outputs": "4f3451469b080548fd0f2ca414a81d91bd0d5690c34378376433ab1ae960ce5c",
+        "ingest-outputs": "b5c8354ebfb71f5d50322c19787bbee2c5bb8fff5bfff46dc5683d19e94797a3",
         "ingest-package-policies": "7817460ef093393f7fce067f91186003e41246a69f32344b1cd68fe19f43a24e",
         "ingest_manager_settings": "6cd91fe6c52c516676d99021f51a4b3a162686880198ba3c556983f5fffbb5a3",
         "integration-config": "b5fba732f6835532e971c81f5703c3c2018fa93686e83a5c2dd9c56cb4e10578",
@@ -851,8 +851,9 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-download-sources|10.1.0: 7fce3be244f92bb99b17c8757c39a49aec078ff90cae70971e42e202a574348c",
         "================================================================================================",
         "ingest-outputs|global: 3e72116f17fda6ec9c5269cb42eb42e3f686b313",
-        "ingest-outputs|mappings: a88cd423056155f954a3877a082c0f8d1273468a",
+        "ingest-outputs|mappings: bdde67dc29cfd86a1195ed34a4417073728691c4",
         "ingest-outputs|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709",
+        "ingest-outputs|10.9.0: 50a325af05e7b9b3f7383f1e525388c9cecf00a14b9924409a262f6c3058840b",
         "ingest-outputs|10.8.0: 7fce3be244f92bb99b17c8757c39a49aec078ff90cae70971e42e202a574348c",
         "ingest-outputs|10.7.0: 2f5dfca42d489deaa60da45c020e92fdae21e5e3d0e5e60153c0d18039715bd7",
         "ingest-outputs|10.6.0: f224697a876b316945536829900df70c8c6af40c28ccd1b1be4854a1df7442dc",
@@ -1485,7 +1486,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "10.0.0",
         "ingest-agent-policies": "10.10.0",
         "ingest-download-sources": "10.1.0",
-        "ingest-outputs": "10.8.0",
+        "ingest-outputs": "10.9.0",
         "ingest-package-policies": "10.22.0",
         "ingest_manager_settings": "10.8.0",
         "integration-config": "10.2.0",
@@ -1650,7 +1651,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "7.16.2",
         "ingest-agent-policies": "10.10.0",
         "ingest-download-sources": "10.1.0",
-        "ingest-outputs": "10.8.0",
+        "ingest-outputs": "10.9.0",
         "ingest-package-policies": "10.22.0",
         "ingest_manager_settings": "10.8.0",
         "integration-config": "10.2.0",

x-pack/platform/plugins/shared/encrypted_saved_objects/integration_tests/ci_checks/check_registered_types.test.ts

@@ -140,6 +140,7 @@ describe('checking changes on all registered encrypted SO types', () => {
         "fleet-fleet-server-host|1",
         "fleet-uninstall-tokens|1",
         "ingest-download-sources|1",
+        "ingest-outputs|9",
         "ingest-outputs|8",
         "ingest-outputs|7",
         "ingest-outputs|6",

x-pack/platform/plugins/shared/fleet/common/types/models/output.ts

@@ -54,6 +54,7 @@ interface NewBaseOutput {
 
 export interface NewElasticsearchOutput extends NewBaseOutput {
   type: OutputType['Elasticsearch'];
+  otel_exporter_config_yaml?: string | null;
 }
 
 export interface NewRemoteElasticsearchOutput extends NewBaseOutput {

x-pack/platform/plugins/shared/fleet/cypress/e2e/fleet_settings_outputs.cy.ts

@@ -37,11 +37,15 @@ import { login } from '../tasks/login';
 import { visit } from '../tasks/common';
 
 export const fillYamlConfigBox = (query: string) => {
-  cy.get('[data-test-subj="kibanaCodeEditor"] textarea').type(query, { force: true });
+  cy.get(
+    '[data-test-subj="settingsOutputsFlyout.yamlConfigInput"] [data-test-subj="kibanaCodeEditor"] textarea'
+  ).type(query, { force: true });
 };
 
 export const clearYamlConfigBox = () => {
-  cy.get('[data-test-subj="kibanaCodeEditor"] textarea').clear({ force: true });
+  cy.get(
+    '[data-test-subj="settingsOutputsFlyout.yamlConfigInput"] [data-test-subj="kibanaCodeEditor"] textarea'
+  ).clear({ force: true });
 };
 
 describe('Outputs', () => {

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.test.tsx

@@ -481,6 +481,137 @@ describe('EditOutputFlyout', () => {
     });
   });
 
+  describe('OpenTelemetry Exporter section', () => {
+    it('should show the OTel exporter configuration section for ES output', async () => {
+      const { utils } = renderFlyout({
+        type: 'elasticsearch',
+        name: 'elasticsearch output',
+        id: 'output123',
+        is_default: false,
+        is_default_monitoring: false,
+      });
+
+      expect(utils.queryByText('OpenTelemetry exporter')).not.toBeNull();
+
+      // Expand the accordion to reveal the YAML editor
+      fireEvent.click(utils.getByText('OpenTelemetry exporter'));
+      expect(utils.queryByLabelText('Advanced YAML configuration')).not.toBeNull();
+    });
+
+    it('should not show the OTel exporter section for logstash output', async () => {
+      const { utils } = renderFlyout({
+        type: 'logstash',
+        name: 'logstash output',
+        id: 'output123',
+        is_default: false,
+        is_default_monitoring: false,
+      });
+
+      expect(utils.queryByText('OpenTelemetry exporter')).toBeNull();
+      expect(utils.queryByLabelText('Advanced YAML Configuration')).toBeNull();
+    });
+
+    it('should not show the OTel exporter section for kafka output', async () => {
+      const { utils } = renderFlyout({
+        type: 'kafka',
+        name: 'kafka output',
+        id: 'output123',
+        is_default: false,
+        is_default_monitoring: false,
+      });
+
+      expect(utils.queryByText('OpenTelemetry exporter')).toBeNull();
+      expect(utils.queryByLabelText('Advanced YAML Configuration')).toBeNull();
+    });
+
+    it('should not show the OTel exporter section for remote ES output', async () => {
+      jest.spyOn(licenseService, 'isEnterprise').mockReturnValue(true);
+      jest
+        .spyOn(ExperimentalFeaturesService, 'get')
+        .mockReturnValue({ enableSyncIntegrationsOnRemote: true } as any);
+
+      const { utils } = renderFlyout({
+        type: 'remote_elasticsearch',
+        name: 'remote es output',
+        id: 'outputR',
+        is_default: false,
+        is_default_monitoring: false,
+      });
+
+      expect(utils.queryByText('OpenTelemetry exporter')).toBeNull();
+      expect(utils.queryByLabelText('Advanced YAML Configuration')).toBeNull();
+    });
+
+    it('should include otel_exporter_config_yaml in the save payload when creating an ES output', async () => {
+      jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({} as any);
+      mockedUseFleetStatus.mockReturnValue({
+        isLoading: false,
+        isReady: true,
+        isSecretsStorageEnabled: true,
+      } as any);
+
+      const { utils } = renderFlyout({
+        type: 'elasticsearch',
+        name: 'elasticsearch output',
+        id: 'output123',
+        is_default: false,
+        is_default_monitoring: false,
+        hosts: ['http://localhost:9200'],
+        otel_exporter_config_yaml: 'flush_interval: 10s',
+      });
+
+      // Change a field so the Save button becomes enabled
+      fireEvent.change(utils.getByDisplayValue('elasticsearch output'), {
+        target: { value: 'updated output name' },
+      });
+
+      fireEvent.click(utils.getByText('Save and apply settings'));
+
+      await waitFor(() => {
+        expect(mockSendPutOutput).toHaveBeenCalledWith(
+          'output123',
+          expect.objectContaining({
+            otel_exporter_config_yaml: 'flush_interval: 10s',
+          })
+        );
+      });
+    });
+
+    it('should send null otel_exporter_config_yaml when the field is empty', async () => {
+      jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({} as any);
+      mockedUseFleetStatus.mockReturnValue({
+        isLoading: false,
+        isReady: true,
+        isSecretsStorageEnabled: true,
+      } as any);
+
+      const { utils } = renderFlyout({
+        type: 'elasticsearch',
+        name: 'elasticsearch output',
+        id: 'output123',
+        is_default: false,
+        is_default_monitoring: false,
+        hosts: ['http://localhost:9200'],
+      });
+
+      // Change a field so the Save button becomes enabled
+      fireEvent.change(utils.getByDisplayValue('elasticsearch output'), {
+        target: { value: 'updated output name' },
+      });
+
+      fireEvent.click(utils.getByText('Save and apply settings'));
+
+      await waitFor(() => {
+        expect(mockSendPutOutput).toHaveBeenCalledWith(
+          'output123',
+          expect.objectContaining({
+            otel_exporter_config_yaml: null,
+          })
+        );
+      });
+    });
+  });
+
   it('should not display remote ES output in type lists if serverless', async () => {
     jest.spyOn(ExperimentalFeaturesService, 'get').mockReturnValue({} as any);
     mockUseStartServices.mockReset();

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx

@@ -32,6 +32,7 @@ import {
   EuiAccordion,
   EuiCode,
   useGeneratedHtmlId,
+  EuiPanel,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 
@@ -330,7 +331,6 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
           </EuiFormRow>
 
           {renderOutputTypeSection(inputs.typeInput.value)}
-
           {isRemoteESOutput ? null : (
             <EuiFormRow
               fullWidth
@@ -370,6 +370,7 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
               </>
             </EuiFormRow>
           )}
+
           <EuiFormRow fullWidth {...inputs.defaultOutputInput.formRowProps}>
             <EuiSwitch
               {...inputs.defaultOutputInput.props}
@@ -547,8 +548,6 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
                 </EuiCallOut>
               </>
             )}
-
-          <EuiSpacer size="l" />
           <EuiFormRow
             label={
               <EuiLink href={docLinks.links.fleet.esSettings} external target="_blank">
@@ -560,26 +559,81 @@ export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> =
             {...inputs.additionalYamlConfigInput.formRowProps}
             fullWidth
           >
-            <YamlCodeEditorWithPlaceholder
-              value={inputs.additionalYamlConfigInput.value}
-              onChange={(value) => {
-                if (outputYmlIncludesReservedPerformanceKey(value, load)) {
-                  inputs.presetInput.setValue('custom');
-                }
+            <div data-test-subj="settingsOutputsFlyout.yamlConfigInput">
+              <YamlCodeEditorWithPlaceholder
+                value={inputs.additionalYamlConfigInput.value}
+                onChange={(value) => {
+                  if (outputYmlIncludesReservedPerformanceKey(value, load)) {
+                    inputs.presetInput.setValue('custom');
+                  }
 
-                inputs.additionalYamlConfigInput.setValue(value);
-              }}
-              disabled={inputs.additionalYamlConfigInput.props.disabled}
-              placeholder={i18n.translate(
-                'xpack.fleet.settings.editOutputFlyout.yamlConfigInputPlaceholder',
-                {
-                  defaultMessage:
-                    '# YAML settings here will be added to the output section of each agent policy.',
-                }
-              )}
-            />
+                  inputs.additionalYamlConfigInput.setValue(value);
+                }}
+                disabled={inputs.additionalYamlConfigInput.props.disabled}
+                placeholder={i18n.translate(
+                  'xpack.fleet.settings.editOutputFlyout.yamlConfigInputPlaceholder',
+                  {
+                    defaultMessage:
+                      '# YAML settings here will be added to the output section of each agent policy.',
+                  }
+                )}
+              />
+            </div>
           </EuiFormRow>
           <AdvancedOptionsSection enabled={form.isShipperEnabled} inputs={inputs} />
+          {isESOutput && (
+            <>
+              <EuiSpacer size="l" />
+              <EuiAccordion
+                id="FleetEditOutputFlyoutOtelExporterConfigAccordion"
+                buttonContent={
+                  <EuiTitle size="xs">
+                    <h3>
+                      <FormattedMessage
+                        id="xpack.fleet.settings.editOutputFlyout.otelExporterConfigTitle"
+                        defaultMessage="OpenTelemetry exporter"
+                      />
+                    </h3>
+                  </EuiTitle>
+                }
+                paddingSize="none"
+              >
+                <EuiPanel color="subdued" borderRadius="none" hasShadow={false} paddingSize="m">
+                  <EuiFormRow
+                    fullWidth
+                    label={
+                      <FormattedMessage
+                        id="xpack.fleet.settings.editOutputFlyout.otelExporterConfigLabel"
+                        defaultMessage="Advanced YAML configuration"
+                      />
+                    }
+                    helpText={
+                      <FormattedMessage
+                        id="xpack.fleet.settings.editOutputFlyout.otelExporterConfigHelpText"
+                        defaultMessage="Settings added here are merged into the Elasticsearch exporter configuration of any OTel-based agent policy that uses this output."
+                      />
+                    }
+                    {...inputs.otelExporterConfigInput.formRowProps}
+                  >
+                    <YamlCodeEditorWithPlaceholder
+                      value={inputs.otelExporterConfigInput.value}
+                      onChange={(value) => inputs.otelExporterConfigInput.setValue(value)}
+                      disabled={inputs.otelExporterConfigInput.props.disabled}
+                      placeholder={i18n.translate(
+                        'xpack.fleet.settings.editOutputFlyout.otelExporterConfigPlaceholder',
+                        {
+                          defaultMessage:
+                            '# YAML settings defined here will be added to the exporter section of OTel policies.',
+                        }
+                      )}
+                    />
+                  </EuiFormRow>
+                </EuiPanel>
+              </EuiAccordion>
+            </>
+          )}
+
+          <EuiSpacer size="l" />
         </EuiForm>
         {output?.id && output.type === 'remote_elasticsearch' ? (
           <OutputHealth output={output} />