feat: multi-instance app status

.changeset/nine-paws-sit.md

@@ -0,0 +1,16 @@
+---
+'@rocket.chat/network-broker': minor
+'@rocket.chat/mock-providers': minor
+'@rocket.chat/pdf-worker': minor
+'@rocket.chat/core-services': minor
+'@rocket.chat/model-typings': minor
+'@rocket.chat/core-typings': minor
+'@rocket.chat/rest-typings': minor
+'@rocket.chat/ui-contexts': minor
+'@rocket.chat/ui-voip': minor
+'@rocket.chat/models': minor
+'@rocket.chat/i18n': minor
+'@rocket.chat/meteor': minor
+---
+
+Enhances the `/api/apps/installed` and `/api/apps/:id/status` endpoints so they get apps' status across the cluster in High-Availability and Microservices deployments

apps/meteor/ee/lib/misc/fetchAppsStatusFromCluster.ts

@@ -0,0 +1,12 @@
+import { Apps } from '@rocket.chat/core-services';
+
+import { isRunningMs } from '../../../server/lib/isRunningMs';
+import { Instance } from '../../server/sdk';
+
+export async function fetchAppsStatusFromCluster() {
+	if (isRunningMs()) {
+		return Apps.getAppsStatusInNodes();
+	}
+
+	return Instance.getAppsStatusInInstances();
+}

apps/meteor/ee/lib/misc/formatAppInstanceForRest.ts

@@ -3,6 +3,8 @@ import type { IAppInfo } from '@rocket.chat/apps-engine/definition/metadata';
 import type { ProxiedApp } from '@rocket.chat/apps-engine/server/ProxiedApp';
 import type { AppLicenseValidationResult } from '@rocket.chat/apps-engine/server/marketplace/license';
 import type { IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
+import type { AppStatusReport } from '@rocket.chat/core-services';
+import type { App } from '@rocket.chat/core-typings';
 
 import { getInstallationSourceFromAppStorageItem } from '../../../lib/apps/getInstallationSourceFromAppStorageItem';
 
@@ -12,9 +14,10 @@ interface IAppInfoRest extends IAppInfo {
 	licenseValidation?: AppLicenseValidationResult;
 	private: boolean;
 	migrated: boolean;
+	clusterStatus?: App['clusterStatus'];
 }
 
-export async function formatAppInstanceForRest(app: ProxiedApp): Promise<IAppInfoRest> {
+export async function formatAppInstanceForRest(app: ProxiedApp, clusterStatus?: AppStatusReport): Promise<IAppInfoRest> {
 	const appRest: IAppInfoRest = {
 		...app.getInfo(),
 		status: await app.getStatus(),
@@ -23,6 +26,10 @@ export async function formatAppInstanceForRest(app: ProxiedApp): Promise<IAppInf
 		migrated: !!app.getStorageItem().migrated,
 	};
 
+	if (clusterStatus?.[app.getID()]) {
+		appRest.clusterStatus = clusterStatus[app.getID()];
+	}
+
 	const licenseValidation = app.getLatestLicenseValidationResult();
 
 	if (licenseValidation?.hasErrors || licenseValidation?.hasWarnings) {

apps/meteor/ee/server/apps/communication/rest.ts

@@ -2,6 +2,7 @@ import { AppStatus, AppStatusUtils } from '@rocket.chat/apps-engine/definition/A
 import type { IAppInfo } from '@rocket.chat/apps-engine/definition/metadata';
 import type { AppManager } from '@rocket.chat/apps-engine/server/AppManager';
 import type { IMarketplaceInfo } from '@rocket.chat/apps-engine/server/marketplace';
+import type { AppStatusReport } from '@rocket.chat/core-services';
 import type { IUser, IMessage } from '@rocket.chat/core-typings';
 import { License } from '@rocket.chat/license';
 import { Settings, Users } from '@rocket.chat/models';
@@ -22,6 +23,7 @@ import { Info } from '../../../../app/utils/rocketchat.info';
 import { i18n } from '../../../../server/lib/i18n';
 import { sendMessagesToAdmins } from '../../../../server/lib/sendMessagesToAdmins';
 import { canEnableApp } from '../../../app/license/server/canEnableApp';
+import { fetchAppsStatusFromCluster } from '../../../lib/misc/fetchAppsStatusFromCluster';
 import { formatAppInstanceForRest } from '../../../lib/misc/formatAppInstanceForRest';
 import { notifyAppInstall } from '../marketplace/appInstall';
 import { fetchMarketplaceApps } from '../marketplace/fetchMarketplaceApps';
@@ -204,7 +206,14 @@ export class AppsRestApi {
 			{
 				async get() {
 					const apps = await manager.get();
-					const formatted = await Promise.all(apps.map(formatAppInstanceForRest));
+					let clusterStatus: AppStatusReport | undefined;
+
+					if (this.queryParams.includeClusterStatus === 'true') {
+						clusterStatus = await fetchAppsStatusFromCluster();
+					}
+
+					const formatted = await Promise.all(apps.map((app) => formatAppInstanceForRest(app, clusterStatus)));
+
 					return API.v1.success({ apps: formatted });
 				},
 			},
@@ -298,7 +307,7 @@ export class AppsRestApi {
 					apiDeprecationLogger.endpoint(this.request.route, '7.0.0', this.response, 'Use /apps/installed to get the installed apps list.');
 
 					const proxiedApps = await manager.get();
-					const apps = await Promise.all(proxiedApps.map(formatAppInstanceForRest));
+					const apps = await Promise.all(proxiedApps.map((app) => formatAppInstanceForRest(app)));
 
 					return API.v1.success({ apps });
 				},
@@ -1268,12 +1277,25 @@ export class AppsRestApi {
 			{ authRequired: true, permissionsRequired: ['manage-apps'] },
 			{
 				async get() {
-					const prl = manager.getOneById(this.urlParams.id);
+					const app = manager.getOneById(this.urlParams.id);
 
-					if (prl) {
-						return API.v1.success({ status: await prl.getStatus() });
+					if (!app) {
+						return API.v1.notFound(`No App found by the id of: ${this.urlParams.id}`);
 					}
-					return API.v1.notFound(`No App found by the id of: ${this.urlParams.id}`);
+
+					const response: { status: AppStatus; clusterStatus?: AppStatusReport[string] } = { status: await app.getStatus() };
+
+					try {
+						const clusterStatus = await fetchAppsStatusFromCluster();
+
+						if (clusterStatus?.[app.getID()]) {
+							response.clusterStatus = clusterStatus[app.getID()];
+						}
+					} catch (e) {
+						orchestrator.getRocketChatLogger().warn('App status endpoint: could not fetch status across cluster', e);
+					}
+
+					return API.v1.success(response);
 				},
 				async post() {
 					const { id: appId } = this.urlParams;

apps/meteor/ee/server/local-services/instance/service.ts

@@ -1,6 +1,7 @@
 import os from 'os';
 
-import { License, ServiceClassInternal } from '@rocket.chat/core-services';
+import type { AppStatusReport } from '@rocket.chat/core-services';
+import { Apps, License, ServiceClassInternal } from '@rocket.chat/core-services';
 import { InstanceStatus, defaultPingInterval, indexExpire } from '@rocket.chat/instance-status';
 import { InstanceStatus as InstanceStatusRaw } from '@rocket.chat/models';
 import EJSON from 'ejson';
@@ -117,6 +118,11 @@ export class InstanceService extends ServiceClassInternal implements IInstanceSe
 					}
 				},
 			},
+			actions: {
+				getAppsStatus(_ctx) {
+					return Apps.getAppsStatusLocal();
+				},
+			},
 		});
 	}
 
@@ -176,4 +182,45 @@ export class InstanceService extends ServiceClassInternal implements IInstanceSe
 	async getInstances(): Promise<BrokerNode[]> {
 		return this.broker.call('$node.list', { onlyAvailable: true });
 	}
+
+	async getAppsStatusInInstances(): Promise<AppStatusReport> {
+		const instances = await this.getInstances();
+
+		const control: Promise<void>[] = [];
+		const statusByApp: AppStatusReport = {};
+
+		instances.forEach((instance) => {
+			if (instance.local) {
+				return;
+			}
+
+			const { id: instanceId } = instance;
+
+			control.push(
+				(async () => {
+					const appsStatus = await this.broker.call<Awaited<ReturnType<(typeof Apps)['getAppsStatusLocal']>>, null>(
+						'matrix.getAppsStatus',
+						null,
+						{ nodeID: instanceId },
+					);
+
+					if (!appsStatus) {
+						throw new Error(`Failed to get apps status from instance ${instanceId}`);
+					}
+
+					appsStatus.forEach(({ status, appId }) => {
+						if (!statusByApp[appId]) {
+							statusByApp[appId] = [];
+						}
+
+						statusByApp[appId].push({ instanceId, status });
+					});
+				})(),
+			);
+		});
+
+		await Promise.all(control);
+
+		return statusByApp;
+	}
 }

apps/meteor/ee/server/sdk/types/IInstanceService.ts

@@ -1,5 +1,7 @@
+import type { AppStatusReport } from '@rocket.chat/core-services';
 import type { BrokerNode } from 'moleculer';
 
 export interface IInstanceService {
 	getInstances(): Promise<BrokerNode[]>;
+	getAppsStatusInInstances(): Promise<AppStatusReport>;
 }

apps/meteor/server/services/apps-engine/service.ts

@@ -4,9 +4,10 @@ import { AppStatusUtils } from '@rocket.chat/apps-engine/definition/AppStatus';
 import type { IAppInfo } from '@rocket.chat/apps-engine/definition/metadata';
 import type { IGetAppsFilter } from '@rocket.chat/apps-engine/server/IGetAppsFilter';
 import type { IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
-import type { IAppsEngineService } from '@rocket.chat/core-services';
+import type { AppStatusReport, IAppsEngineService } from '@rocket.chat/core-services';
 import { ServiceClassInternal } from '@rocket.chat/core-services';
 
+import { isRunningMs } from '../../lib/isRunningMs';
 import { SystemLogger } from '../../lib/logger/system';
 
 export class AppsEngineService extends ServiceClassInternal implements IAppsEngineService {
@@ -133,4 +134,68 @@ export class AppsEngineService extends ServiceClassInternal implements IAppsEngi
 
 		return app.getStorageItem();
 	}
+
+	async getAppsStatusLocal(): Promise<{ status: AppStatus; appId: string }[]> {
+		const apps = await Apps.self?.getManager()?.get();
+
+		if (!apps) {
+			return [];
+		}
+
+		return Promise.all(
+			apps.map(async (app) => ({
+				status: await app.getStatus(),
+				appId: app.getID(),
+			})),
+		);
+	}
+
+	async getAppsStatusInNodes(): Promise<AppStatusReport> {
+		if (!isRunningMs()) {
+			throw new Error('Getting apps status in cluster is only available in microservices mode');
+		}
+
+		if (!this.api) {
+			throw new Error('AppsEngineService is not initialized');
+		}
+
+		// If we are running MS AND this.api is defined, we KNOW there is a local node
+		/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
+		const { id: localNodeId } = (await this.api.nodeList()).find((node) => node.local)!;
+
+		const services: { name: string; nodes: string[] }[] = await this.api?.call('$node.services', { onlyActive: true });
+
+		// We can filter out the local node because we already know its status
+		const availableNodes = services?.find((service) => service.name === 'apps-engine')?.nodes.filter((node) => node !== localNodeId);
+
+		if (!availableNodes || availableNodes.length < 1) {
+			throw new Error('Not enough Apps-Engine nodes in deployment');
+		}
+
+		const statusByApp: AppStatusReport = {};
+
+		const apps: Promise<void>[] = availableNodes.map(async (nodeID) => {
+			const appsStatus: Awaited<ReturnType<typeof this.getAppsStatusLocal>> | undefined = await this.api?.call(
+				'apps-engine.getAppsStatusLocal',
+				[],
+				{ nodeID },
+			);
+
+			if (!appsStatus) {
+				throw new Error(`Failed to get apps status from node ${nodeID}`);
+			}
+
+			appsStatus.forEach(({ status, appId }) => {
+				if (!statusByApp[appId]) {
+					statusByApp[appId] = [];
+				}
+
+				statusByApp[appId].push({ instanceId: nodeID, status });
+			});
+		});
+
+		await Promise.all(apps);
+
+		return statusByApp;
+	}
 }