[9.1] [UII] Use explicit fleet server & output IDs for agentless on ECH and Serverless (#225188)

x-pack/platform/plugins/shared/fleet/common/constants/fleet_server_policy_config.ts

@@ -14,3 +14,12 @@ export const FLEET_PROXY_SAVED_OBJECT_TYPE = 'fleet-proxy';
 export const PROXY_URL_REGEX = /^(http[s]?|socks5):\/\/[^\s$.?#].[^\s]*$/gm;
 
 export const SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID = 'default-fleet-server';
+
+// Fleet Server IDs used for agentless policies:
+//  - For ECH, this is created by Fleet, see `createCloudFleetServerHostsIfNeeded` in:
+//    `x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/fleet_server_host.ts`
+//  - For Serverless, this is the `default-fleet-server` host that is created from
+//    preconfiguration via project controller
+//  - Both are uneditable by users due to having `is_preconfigured: true` set
+export const ECH_AGENTLESS_FLEET_SERVER_HOST_ID = 'internal-agentless-fleet-server';
+export const SERVERLESS_AGENTLESS_FLEET_SERVER_HOST_ID = SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID;

x-pack/platform/plugins/shared/fleet/common/constants/output.ts

@@ -28,6 +28,15 @@ export const DEFAULT_OUTPUT: NewOutput = {
 
 export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output';
 
+// Output IDs used for agentless policies:
+//  - For ECH, this is an output created by Fleet, see `ensureDefaultOutputs()` in
+//    `x-pack/plugins/fleet/server/services/output.ts`
+//  - For Serverless, this is the `es-default-output` output that is created from
+//    preconfiguration via project controller
+//  - Both are uneditable by users due to having `is_preconfigured: true` set
+export const ECH_AGENTLESS_OUTPUT_ID = 'es-agentless-output';
+export const SERVERLESS_AGENTLESS_OUTPUT_ID = SERVERLESS_DEFAULT_OUTPUT_ID;
+
 export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum';
 export const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise';
 

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.ts

@@ -21,10 +21,10 @@ import { SelectedPolicyTab } from '../../components';
 import {
   AGENTLESS_AGENT_POLICY_INACTIVITY_TIMEOUT,
   AGENTLESS_AGENT_POLICY_MONITORING,
-  SERVERLESS_DEFAULT_OUTPUT_ID,
-  DEFAULT_OUTPUT_ID,
-  SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID,
-  DEFAULT_FLEET_SERVER_HOST_ID,
+  ECH_AGENTLESS_OUTPUT_ID,
+  SERVERLESS_AGENTLESS_OUTPUT_ID,
+  ECH_AGENTLESS_FLEET_SERVER_HOST_ID,
+  SERVERLESS_AGENTLESS_FLEET_SERVER_HOST_ID,
 } from '../../../../../../../../common/constants';
 import {
   isAgentlessIntegration as isAgentlessIntegrationFn,
@@ -153,9 +153,9 @@ export function useSetupTechnology({
   useEffect(() => {
     const fetchOutputId = async () => {
       const outputId = isServerless
-        ? SERVERLESS_DEFAULT_OUTPUT_ID
+        ? SERVERLESS_AGENTLESS_OUTPUT_ID
         : isCloud
-        ? DEFAULT_OUTPUT_ID
+        ? ECH_AGENTLESS_OUTPUT_ID
         : undefined;
       if (outputId) {
         const outputData = await sendGetOneOutput(outputId);
@@ -166,9 +166,9 @@ export function useSetupTechnology({
     };
     const fetchFleetServerHostId = async () => {
       const hostId = isServerless
-        ? SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID
+        ? SERVERLESS_AGENTLESS_FLEET_SERVER_HOST_ID
         : isCloud
-        ? DEFAULT_FLEET_SERVER_HOST_ID
+        ? ECH_AGENTLESS_FLEET_SERVER_HOST_ID
         : undefined;
 
       if (hostId) {
@@ -214,7 +214,12 @@ export function useSetupTechnology({
         inactivity_timeout: AGENTLESS_AGENT_POLICY_INACTIVITY_TIMEOUT,
         supports_agentless: true,
         monitoring_enabled: AGENTLESS_AGENT_POLICY_MONITORING,
-        ...(agentlessPolicyOutputId ? { data_output_id: agentlessPolicyOutputId } : {}),
+        ...(agentlessPolicyOutputId
+          ? {
+              data_output_id: agentlessPolicyOutputId,
+              monitoring_output_id: agentlessPolicyOutputId,
+            }
+          : {}),
         ...(agentlessPolicyFleetServerHostId
           ? { fleet_server_host_id: agentlessPolicyFleetServerHostId }
           : {}),

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/index.tsx

@@ -63,20 +63,20 @@ import { OutputFormLogstashSection } from './output_form_logstash';
 import { OutputFormElasticsearchSection } from './output_form_elasticsearch';
 
 export interface EditOutputFlyoutProps {
-  defaultOuput?: Output;
+  defaultOutput?: Output;
   output?: Output;
   onClose: () => void;
   proxies: FleetProxy[];
 }
 
 export const EditOutputFlyout: React.FunctionComponent<EditOutputFlyoutProps> = ({
-  defaultOuput,
+  defaultOutput,
   onClose,
   output,
   proxies,
 }) => {
   useBreadcrumbs('settings');
-  const form = useOutputForm(onClose, output, defaultOuput);
+  const form = useOutputForm(onClose, output, defaultOutput);
   const inputs = form.inputs;
   const { docLinks, cloud } = useStartServices();
   const fleetStatus = useFleetStatus();

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx

@@ -193,7 +193,7 @@ export function extractDefaultDynamicKafkaTopics(
   ];
 }
 
-export function useOutputForm(onSucess: () => void, output?: Output, defaultOuput?: Output) {
+export function useOutputForm(onSucess: () => void, output?: Output, defaultOutput?: Output) {
   const fleetStatus = useFleetStatus();
   const authz = useAuthz();
 
@@ -254,7 +254,7 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu
   const isServerless = cloud?.isServerlessEnabled;
   // Set the hosts to default for new ES output in serverless.
   const elasticsearchUrlDefaultValue =
-    isServerless && !output?.hosts ? defaultOuput?.hosts || [] : output?.hosts || [];
+    isServerless && !output?.hosts ? defaultOutput?.hosts || [] : output?.hosts || [];
   const elasticsearchUrlDisabled = isServerless || isDisabled('hosts');
   const elasticsearchUrlInput = useComboInput(
     'esHostsComboxBox',

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/index.tsx

@@ -84,6 +84,7 @@ export const SettingsApp = withConfirmModalProvider(() => {
   ]);
 
   const { cloud } = useStartServices();
+  const isServerlessEnabled = cloud?.isServerlessEnabled;
 
   if (
     (outputs.isLoading && outputs.isInitialRequest) ||
@@ -127,7 +128,7 @@ export const SettingsApp = withConfirmModalProvider(() => {
         </Route>
         <Route path={FLEET_ROUTING_PATHS.settings_create_fleet_server_hosts}>
           <EuiPortal>
-            {cloud?.isServerlessEnabled ? (
+            {isServerlessEnabled ? (
               <FleetServerHostsFlyout
                 proxies={proxies.data?.items ?? []}
                 onClose={onCloseCallback}
@@ -145,7 +146,11 @@ export const SettingsApp = withConfirmModalProvider(() => {
             <EditOutputFlyout
               proxies={proxies.data.items}
               onClose={onCloseCallback}
-              defaultOuput={outputs.data?.items.find((o) => o.id === SERVERLESS_DEFAULT_OUTPUT_ID)}
+              defaultOutput={
+                isServerlessEnabled
+                  ? outputs.data?.items.find((o) => o.id === SERVERLESS_DEFAULT_OUTPUT_ID)
+                  : undefined
+              }
             />
           </EuiPortal>
         </Route>
@@ -182,9 +187,11 @@ export const SettingsApp = withConfirmModalProvider(() => {
                   proxies={proxies.data?.items ?? []}
                   onClose={onCloseCallback}
                   output={output}
-                  defaultOuput={outputs.data?.items.find(
-                    (o) => o.id === SERVERLESS_DEFAULT_OUTPUT_ID
-                  )}
+                  defaultOutput={
+                    isServerlessEnabled
+                      ? outputs.data?.items.find((o) => o.id === SERVERLESS_DEFAULT_OUTPUT_ID)
+                      : undefined
+                  }
                 />
               </EuiPortal>
             );

x-pack/platform/plugins/shared/fleet/server/constants/index.ts

@@ -63,6 +63,8 @@ export {
   DEFAULT_OUTPUT,
   DEFAULT_OUTPUT_ID,
   SERVERLESS_DEFAULT_OUTPUT_ID,
+  ECH_AGENTLESS_OUTPUT_ID,
+  SERVERLESS_AGENTLESS_OUTPUT_ID,
   PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
   AGENT_POLICY_DEFAULT_MONITORING_DATASETS,
   // Fleet Server index
@@ -84,6 +86,8 @@ export {
   DEFAULT_FLEET_SERVER_HOST_ID,
   FLEET_SERVER_HOST_SAVED_OBJECT_TYPE,
   SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID,
+  ECH_AGENTLESS_FLEET_SERVER_HOST_ID,
+  SERVERLESS_AGENTLESS_FLEET_SERVER_HOST_ID,
   FLEET_SERVER_PACKAGE,
   // Proxy
   FLEET_PROXY_SAVED_OBJECT_TYPE,

x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts

@@ -442,7 +442,7 @@ describe('Agent policy', () => {
         updated_by: 'system',
         schema_version: '1.1.1',
         is_protected: false,
-        fleet_server_host_id: 'fleet-default-fleet-server-host',
+        fleet_server_host_id: 'internal-agentless-fleet-server',
       });
     });
 

x-pack/platform/plugins/shared/fleet/server/services/agentless_settings_ids.test.ts

@@ -70,6 +70,7 @@ describe('correct agentless policy settings', () => {
       'agent_policy_1',
       {
         data_output_id: 'es-default-output',
+        monitoring_output_id: 'es-default-output',
         fleet_server_host_id: 'default-fleet-server',
       },
       {
@@ -82,6 +83,7 @@ describe('correct agentless policy settings', () => {
       'agent_policy_2',
       {
         data_output_id: 'es-default-output',
+        monitoring_output_id: 'es-default-output',
         fleet_server_host_id: 'default-fleet-server',
       },
       {

x-pack/platform/plugins/shared/fleet/server/services/agentless_settings_ids.ts

@@ -35,7 +35,7 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
   const internalSoClientWithoutSpaceExtension =
     appContextService.getInternalUserSOClientWithoutSpaceExtension();
 
-  const agentlessOutputIdsToFix = correctOutputId
+  const agentlessDataOutputIdsToFix = correctOutputId
     ? (
         await internalSoClientWithoutSpaceExtension.find<AgentPolicySOAttributes>({
           type: agentPolicySavedObjectType,
@@ -48,6 +48,19 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
       )?.saved_objects.map((so) => so.id)
     : [];
 
+  const agentlessMonitoringOutputIdsToFix = correctOutputId
+    ? (
+        await internalSoClientWithoutSpaceExtension.find<AgentPolicySOAttributes>({
+          type: agentPolicySavedObjectType,
+          page: 1,
+          perPage: SO_SEARCH_LIMIT,
+          filter: `${agentPolicySavedObjectType}.attributes.supports_agentless:true AND NOT ${agentPolicySavedObjectType}.attributes.monitoring_output_id:${correctOutputId}`,
+          fields: [`id`],
+          namespaces: ['*'],
+        })
+      )?.saved_objects.map((so) => so.id)
+    : [];
+
   const agentlessFleetServerIdsToFix = correctFleetServerId
     ? (
         await internalSoClientWithoutSpaceExtension.find<AgentPolicySOAttributes>({
@@ -63,15 +76,18 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
 
   try {
     // Check that the output ID exists
-    if (correctOutputId && agentlessOutputIdsToFix?.length > 0) {
+    if (
+      correctOutputId &&
+      (agentlessDataOutputIdsToFix?.length > 0 || agentlessMonitoringOutputIdsToFix?.length > 0)
+    ) {
       const output = await outputService.get(
         internalSoClientWithoutSpaceExtension,
         correctOutputId
       );
       fixOutput = output != null;
     }
   } catch (e) {
-    // Silently swallow
+    // Silently swallow so that output will not be fixed if the correct output ID does not exist
   }
 
   try {
@@ -84,12 +100,13 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
       fixFleetServer = fleetServerHost != null;
     }
   } catch (e) {
-    // Silently swallow
+    // Silently swallow so that fleet server host will not be fixed if the correct fleet server host ID does not exist
   }
 
   const allIdsToFix = Array.from(
     new Set([
-      ...(fixOutput ? agentlessOutputIdsToFix : []),
+      ...(fixOutput ? agentlessDataOutputIdsToFix : []),
+      ...(fixOutput ? agentlessMonitoringOutputIdsToFix : []),
       ...(fixFleetServer ? agentlessFleetServerIdsToFix : []),
     ])
   );
@@ -100,9 +117,7 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
 
   appContextService
     .getLogger()
-    .debug(
-      `Fixing output and/or fleet server host IDs on agent policies: ${agentlessOutputIdsToFix}`
-    );
+    .debug(`Fixing output and/or fleet server host IDs on agent policies: ${allIdsToFix}`);
 
   await pMap(
     allIdsToFix,
@@ -113,6 +128,7 @@ export async function ensureCorrectAgentlessSettingsIds(esClient: ElasticsearchC
         agentPolicyId,
         {
           data_output_id: correctOutputId,
+          monitoring_output_id: correctOutputId,
           fleet_server_host_id: correctFleetServerId,
         },
         {

x-pack/platform/plugins/shared/fleet/server/services/agents/agentless_agent.ts

@@ -33,9 +33,9 @@ import {
   AGENTLESS_GLOBAL_TAG_NAME_ORGANIZATION,
   AGENTLESS_GLOBAL_TAG_NAME_DIVISION,
   AGENTLESS_GLOBAL_TAG_NAME_TEAM,
-  DEFAULT_OUTPUT_ID,
+  ECH_AGENTLESS_OUTPUT_ID,
+  ECH_AGENTLESS_FLEET_SERVER_HOST_ID,
   SERVERLESS_DEFAULT_OUTPUT_ID,
-  DEFAULT_FLEET_SERVER_HOST_ID,
   SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID,
 } from '../../constants';
 
@@ -70,12 +70,12 @@ class AgentlessAgentService {
     const outputId = isServerless
       ? SERVERLESS_DEFAULT_OUTPUT_ID
       : isCloud
-      ? DEFAULT_OUTPUT_ID
+      ? ECH_AGENTLESS_OUTPUT_ID
       : undefined;
     const fleetServerId = isServerless
       ? SERVERLESS_DEFAULT_FLEET_SERVER_HOST_ID
       : isCloud
-      ? DEFAULT_FLEET_SERVER_HOST_ID
+      ? ECH_AGENTLESS_FLEET_SERVER_HOST_ID
       : undefined;
 
     return {

x-pack/platform/plugins/shared/fleet/server/services/output.ts

@@ -83,6 +83,7 @@ import {
 } from './secrets';
 import { findAgentlessPolicies } from './outputs/helpers';
 import { patchUpdateDataWithRequireEncryptedAADFields } from './outputs/so_helpers';
+
 import {
   canEnableSyncIntegrations,
   createOrUpdateFleetSyncedIntegrationsIndex,