[Fleet] Create API to report status of integrations synchronization

x-pack/platform/plugins/shared/fleet/common/constants/routes.ts

@@ -208,6 +208,10 @@ export const DOWNLOAD_SOURCE_API_ROUTES = {
   DELETE_PATTERN: `${API_ROOT}/agent_download_sources/{sourceId}`,
 };
 
+export const REMOTE_SYNCED_INTEGRATIONS_API_ROUTES = {
+  STATUS_PATTERN: `${API_ROOT}/remote_synced_integrations/status`,
+};
+
 export const CREATE_STANDALONE_AGENT_API_KEY_ROUTE = `${INTERNAL_ROOT}/create_standalone_agent_api_key`;
 
 // Fleet debug routes

x-pack/platform/plugins/shared/fleet/common/types/models/index.ts

@@ -21,3 +21,4 @@ export * from './fleet_proxy';
 export * from './secret';
 export * from './setup_technology';
 export * from './fleet_setup_lock';
+export * from './remote_synced_integrations';

x-pack/platform/plugins/shared/fleet/common/types/models/remote_synced_integrations.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { CustomAssetsData } from '../../../server/tasks/sync_integrations/model';
+
+export enum SyncStatus {
+  SYNCHRONIZING = 'synchronizing',
+  COMPLETED = 'completed',
+  FAILED = 'failed',
+}
+
+export interface RemoteSyncedIntegrationsBase {
+  id?: string;
+  package_name: string;
+  package_version: string;
+}
+export interface RemoteSyncedIntegrationsStatus extends RemoteSyncedIntegrationsBase {
+  sync_status: SyncStatus;
+  error?: string;
+  updated_at?: string;
+}
+export interface RemoteSyncedCustomAssetsStatus extends CustomAssetsData {
+  sync_status: SyncStatus;
+  error?: string;
+}
+
+export interface RemoteSyncedCustomAssetsRecord {
+  [key: string]: RemoteSyncedCustomAssetsStatus;
+}

x-pack/platform/plugins/shared/fleet/common/types/rest_spec/index.ts

@@ -21,3 +21,4 @@ export * from './settings';
 export * from './health_check';
 export * from './fleet_server_hosts';
 export * from './standalone_agent_api_key';
+export * from './remote_synced_integrations';

x-pack/platform/plugins/shared/fleet/common/types/rest_spec/remote_synced_integrations.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { RemoteSyncedCustomAssetsRecord, RemoteSyncedIntegrationsStatus } from '../models';
+
+export interface GetRemoteSyncedIntegrationsStatusResponse {
+  integrations: RemoteSyncedIntegrationsStatus[];
+  custom_assets?: RemoteSyncedCustomAssetsRecord;
+  error?: string;
+}

x-pack/platform/plugins/shared/fleet/dev_docs/local_setup/remote_clusters_ccr.md

@@ -1,93 +1,164 @@
-# Test sync integrations feature 
+# Test sync integrations feature
 
-## Start Cluster 1 (remote)
+## Run ES and Kibana
 
+### Changes to `kibana.dev.yml`
+
+- Remove `elasticsearch.hosts`
+- Enable feature flag
+
+```
+  xpack.fleet.enableExperimental: ['enableSyncIntegrationsOnRemote']
+```
+
+This configuration allows to run two local ES clusters in parallel, each one having its own instance of Kibana.
+
+### Start Cluster 1 (remote)
 - Start ES 1
 
 ```
 yarn es snapshot -E http.port=9500 -E transport.port=9600 -E path.data=../remote --license trial -E http.host=0.0.0.0
 ```
 
+Verify that node is healthy
+
+```
+  curl -k -u elastic:changeme http://localhost:9500
+```
+
 - Start Kibana 1
 
 ```
 yarn start --server.port=5701 --elasticsearch.hosts=http://localhost:9500 --dev.basePathProxyTarget=5703
 ```
+- Login into http://localhost/5601/<YOUR_PATH>
 
-## Start Cluster 2 (main)
+### Start Cluster 2 (main)
 
 - Start ES 2
 
 ```
 yarn es snapshot --license trial -E path.data=/tmp/es-data -E http.host=0.0.0.0
 ```
+Note that `transport.port` defaults to `9300`
+
+Verify that node is healthy:
+
+```
+curl -k -u elastic:changeme http://localhost:9200
+```
 
 - Start Kibana 2
 
 ```
 yarn start
 ```
 
-## Enable feature flag
+- Login into http://localhost/5701/<YOUR_PATH>
+
+To avoid issues, it might be needed to login to one of the kibana instances with an incognito session.
 
-- Add to `kibana.dev.yml`
+## Setup on Cluster 1
 
+- Navigate to dev tools and create service token and api key
 ```
-xpack.fleet.enableExperimental: ['enableSyncIntegrationsOnRemote']
+POST kbn:/api/fleet/service_tokens
+{
+  "remote": true
+}
+
+POST /_security/api_key
+  {
+    "name": "integration_sync_api_key",
+    "role_descriptors": {
+      "integration_writer": {
+        "cluster": [],
+      "indices":[],
+      "applications": [{
+          "application": "kibana-.kibana",
+            "privileges": ["feature_fleet.read", "feature_fleetv2.read"],
+            "resources": ["*"]
+        }]
+      }
+    }
+  }
 ```
+Save the responses as they will be required in Cluster 2 (see next section).
+
+### Add Remote Cluster
+
+- Add Remote Cluster to Cluster 1. Navigate to *Stack Management > Remote Clusters* ([link](http://localhost:5601/app/management/data/remote_clusters)) and follow the steps to add cluster 'local':
+
+- Click "Add a remote cluster"
+- Choose a name, put `localhost:9300` for "Seed nodes", and save (check "Yes, I have setup trust")
+- Make sure the connection status is "Connected"
 
-## Add remote ES output
-- Add remote ES output to Cluster 2
+### Set up CCR
+
+Please note that [CCR](https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-getting-started-tutorial.html) requires both clusters to have the same license. At the time of writing an `enterprise` license is needed.
+On cluster 1, navigate to *Stack Management > Cross-Cluster Replication* and create a follower index using the cluster from Step 1.
+
+  - Leader index `fleet-synced-integrations`
+  - Follower index `fleet-synced-integrations-ccr-remote1`
+
+### Set up local ES output
+This configuration is required to kick off the integration sync. The local host needs to match the remote ES output configured on Cluster 2 (see next section). Note that `kibana.dev.yml` is read by both kibana instances so it's better to add it in the UI to avoid conflicts.
 
 ```
-xpack.fleet.outputs:
-  - name: 'Preconfiged remote output'
-    type: 'remote_elasticsearch'
-    id: 'remote-output'
-    hosts: ["http://<local_ip>:9500"]
-    sync_integrations: true
-    kibana_url: "http://localhost:5701"
-    secrets:
-      service_token: token
-      kibana_api_key: key
+  name: 'ES output'
+  type: 'elasticsearch'
+  id: 'local-output'
+  hosts: ["http://<local_ip>:9500"]
 ```
 
-## Add Remote Cluster
+## Setup on Cluster 2
 
-- Add Remote Cluster to Cluster 1
+### Add remote ES output
+- Add remote ES output to Cluster 2. Insert `token` and `key` obtained from the previous commands.
+```
+  name: 'Remote output'
+  type: 'remote_elasticsearch'
+  id: 'remote-output'
+  hosts: ["http://<local_ip>:9500"]
+  sync_integrations: true
+  kibana_url: "http://localhost:5701"
+  secrets:
+    service_token: token
+    kibana_api_key: key
+```
 
-http://localhost:5701/app/management/data/remote_clusters 
+## Verify sync
 
-Add Remote cluster with remote address `localhost:9300`
+With these configuration, the sync task should run every 5 minutes and the integrations installed on Cluster 1 should be the same as the ones on Cluster 2
 
-## Set up CCR
+### Check contents of sync integrations indices
 
-- Add follower index to Cluster 1
-  - Leader index `fleet-synced-integrations`
-  - Follower index `fleet-synced-integrations-ccr-remote1`
+- From Cluster 1, verify that the follower index is active using the [follower info API](https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-follow-info.html):
 
+```
+GET fleet-synced-integrations-ccr-remote1/_ccr/info
+```
 
-## Check contents of sync integrations indices
+Check content of ccr index:
 
-- In Cluster 1
 ```
 GET fleet-synced-integrations-ccr-remote1/_search
 ```
 
-- In Cluster 2
+- From Cluster 2
 ```
 GET fleet-synced-integrations/_search
 ```
 
-## Read data from remote with CCS
+### Read data from remote with CCS
 
 - Enroll Fleet Server to Cluster 2
 
-- Create Agent Policy in Cluster 2 and use remote output for integrations and monitoring
+- Create an Agent Policy in Cluster 2 and use remote output for integrations and monitoring
 
-- Enroll Agent to Cluster 2 to the Agent policy
+- Enroll an agent the above Agent policy in Cluster 2
 
-- Check data in Cluster 1 with CCS
+- Check data in Cluster 1 with [CCS](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cross-cluster-search.html)
 
 ```
 GET remote1:metrics-*/_search

x-pack/platform/plugins/shared/fleet/server/errors/index.ts

@@ -134,6 +134,7 @@ export class InputNotFoundError extends FleetNotFoundError {}
 export class OutputNotFoundError extends FleetNotFoundError {}
 export class PackageNotFoundError extends FleetNotFoundError {}
 export class ArchiveNotFoundError extends FleetNotFoundError {}
+export class IndexNotFoundError extends FleetNotFoundError {}
 
 export class PackagePolicyNotFoundError extends FleetNotFoundError<{
   /** The package policy ID that was not found */

x-pack/platform/plugins/shared/fleet/server/routes/index.ts

@@ -28,6 +28,7 @@ import { registerRoutes as registerMessageSigningServiceRoutes } from './message
 import { registerRoutes as registerUninstallTokenRoutes } from './uninstall_token';
 import { registerRoutes as registerStandaloneAgentApiKeyRoutes } from './standalone_agent_api_key';
 import { registerRoutes as registerDebugRoutes } from './debug';
+import { registerRoutes as registerRemoteSyncedIntegrations } from './remote_synced_integrations';
 
 export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: FleetConfigType) {
   // Always register app routes for permissions checking
@@ -50,6 +51,7 @@ export function registerRoutes(fleetAuthzRouter: FleetAuthzRouter, config: Fleet
   registerMessageSigningServiceRoutes(fleetAuthzRouter);
   registerUninstallTokenRoutes(fleetAuthzRouter, config);
   registerStandaloneAgentApiKeyRoutes(fleetAuthzRouter);
+  registerRemoteSyncedIntegrations(fleetAuthzRouter);
   registerDebugRoutes(fleetAuthzRouter);
 
   // Conditional config routes

x-pack/platform/plugins/shared/fleet/server/routes/remote_synced_integrations/handler.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RequestHandler } from '@kbn/core/server';
+
+import type { GetRemoteSyncedIntegrationsStatusResponse } from '../../../common/types';
+
+import { getRemoteSyncedIntegrationsStatus } from '../../tasks/sync_integrations/sync_integrations_on_remote';
+
+export const getRemoteSyncedIntegrationsStatusHandler: RequestHandler<undefined> = async (
+  context,
+  request,
+  response
+) => {
+  const coreContext = await context.core;
+  const esClient = coreContext.elasticsearch.client.asInternalUser;
+  const soClient = coreContext.savedObjects.client;
+
+  try {
+    const res: GetRemoteSyncedIntegrationsStatusResponse = await getRemoteSyncedIntegrationsStatus(
+      esClient,
+      soClient
+    );
+
+    return response.ok({ body: res });
+  } catch (error) {
+    throw error;
+  }
+};

x-pack/platform/plugins/shared/fleet/server/routes/remote_synced_integrations/index.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FleetAuthzRouter } from '../../services/security';
+
+import { API_VERSIONS } from '../../../common/constants';
+import { REMOTE_SYNCED_INTEGRATIONS_API_ROUTES } from '../../../common/constants';
+import { FLEET_API_PRIVILEGES } from '../../constants/api_privileges';
+import { genericErrorResponse } from '../schema/errors';
+import { GetRemoteSyncedIntegrationsStatusResponseSchema } from '../../types/models/synced_integrations';
+
+import { getRemoteSyncedIntegrationsStatusHandler } from './handler';
+
+export const registerRoutes = (router: FleetAuthzRouter) => {
+  router.versioned
+    .get({
+      path: REMOTE_SYNCED_INTEGRATIONS_API_ROUTES.STATUS_PATTERN,
+      security: {
+        authz: {
+          requiredPrivileges: [
+            FLEET_API_PRIVILEGES.SETTINGS.READ,
+            FLEET_API_PRIVILEGES.INTEGRATIONS.READ,
+          ],
+        },
+      },
+      summary: `Get CCR Remote synced integrations status`,
+      options: {
+        tags: ['oas-tag:CCR Remote synced integrations'],
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        validate: {
+          request: {},
+          response: {
+            200: {
+              body: () => GetRemoteSyncedIntegrationsStatusResponseSchema,
+            },
+            400: {
+              body: genericErrorResponse,
+            },
+          },
+        },
+      },
+      getRemoteSyncedIntegrationsStatusHandler
+    );
+};

x-pack/platform/plugins/shared/fleet/server/services/setup/fleet_synced_integrations.ts

@@ -11,6 +11,7 @@ import { FleetSetupError } from '../../errors';
 import { appContextService } from '../app_context';
 
 export const FLEET_SYNCED_INTEGRATIONS_INDEX_NAME = 'fleet-synced-integrations';
+export const FLEET_SYNCED_INTEGRATIONS_CCR_INDEX_PREFIX = 'fleet-synced-integrations-ccr-*';
 
 export const FLEET_SYNCED_INTEGRATIONS_INDEX_CONFIG = {
   settings: {