feat: Implement detailed tracing

apps/meteor/app/api/server/api.ts

@@ -1,6 +1,7 @@
 import type { IRoom } from '@rocket.chat/core-typings';
 import type { Router } from '@rocket.chat/http-router';
 import { Logger } from '@rocket.chat/logger';
+import { tracerSpanMiddleware } from '@rocket.chat/tracing';
 import type express from 'express';
 import { WebApp } from 'meteor/webapp';
 
@@ -9,7 +10,6 @@ import { cors } from './middlewares/cors';
 import { loggerMiddleware } from './middlewares/logger';
 import { metricsMiddleware } from './middlewares/metrics';
 import { remoteAddressMiddleware } from './middlewares/remoteAddressMiddleware';
-import { tracerSpanMiddleware } from './middlewares/tracer';
 import { type APIActionHandler, RocketChatAPIRouter } from './router';
 import { metrics } from '../../metrics/server';
 import { settings } from '../../settings/server';
@@ -106,7 +106,7 @@ export const startRestAPI = () => {
 			.use(cors(settings))
 			.use(loggerMiddleware(logger))
 			.use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi }))
-			.use(tracerSpanMiddleware)
+			.use(tracerSpanMiddleware())
 			.use(API.v1.router)
 			.use(API.default.router).router,
 	);

apps/meteor/app/integrations/server/api/api.ts

@@ -2,6 +2,7 @@ import type { IIncomingIntegration, IIntegration, IOutgoingIntegration, IUser, R
 import { Integrations, Users } from '@rocket.chat/models';
 import { Random } from '@rocket.chat/random';
 import { isIntegrationsHooksAddSchema, isIntegrationsHooksRemoveSchema } from '@rocket.chat/rest-typings';
+import { tracerSpanMiddleware } from '@rocket.chat/tracing';
 import type express from 'express';
 import type { Context, Next } from 'hono';
 import { Meteor } from 'meteor/meteor';
@@ -16,7 +17,6 @@ import { API, defaultRateLimiterOptions } from '../../../api/server/api';
 import type { FailureResult, GenericRouteExecutionContext, SuccessResult, UnavailableResult } from '../../../api/server/definition';
 import { loggerMiddleware } from '../../../api/server/middlewares/logger';
 import { metricsMiddleware } from '../../../api/server/middlewares/metrics';
-import { tracerSpanMiddleware } from '../../../api/server/middlewares/tracer';
 import type { WebhookResponseItem } from '../../../lib/server/functions/processWebhookMessage';
 import { processWebhookMessage } from '../../../lib/server/functions/processWebhookMessage';
 import { metrics } from '../../../metrics/server';
@@ -386,7 +386,7 @@ const Api = new WebHookAPI({
 Api.router
 	.use(loggerMiddleware(integrationLogger))
 	.use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi }))
-	.use(tracerSpanMiddleware);
+	.use(tracerSpanMiddleware());
 
 const middleware = async (c: Context, next: Next): Promise<void> => {
 	const { req } = c;

apps/meteor/ee/server/apps/communication/rest.ts

@@ -8,6 +8,7 @@ import { License } from '@rocket.chat/license';
 import { Logger } from '@rocket.chat/logger';
 import { Settings, Users } from '@rocket.chat/models';
 import { serverFetch as fetch } from '@rocket.chat/server-fetch';
+import { tracerSpanMiddleware } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 import * as z from 'zod';
 
@@ -22,7 +23,6 @@ import type { APIClass } from '../../../../app/api/server/ApiClass';
 import { getUploadFormData } from '../../../../app/api/server/lib/getUploadFormData';
 import { loggerMiddleware } from '../../../../app/api/server/middlewares/logger';
 import { metricsMiddleware } from '../../../../app/api/server/middlewares/metrics';
-import { tracerSpanMiddleware } from '../../../../app/api/server/middlewares/tracer';
 import { getWorkspaceAccessToken, getWorkspaceAccessTokenWithScope } from '../../../../app/cloud/server';
 import { metrics } from '../../../../app/metrics/server';
 import { settings } from '../../../../app/settings/server';
@@ -74,7 +74,7 @@ export class AppsRestApi {
 		this.api.router
 			.use(loggerMiddleware(logger))
 			.use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi }))
-			.use(tracerSpanMiddleware);
+			.use(tracerSpanMiddleware());
 
 		this.addManagementRoutes();
 		// Using the same instance of the existing API for now, to be able to use the same api prefix(/api)

ee/packages/federation-matrix/README.md

@@ -25,52 +25,85 @@ The integration test script builds Rocket.Chat locally, starts federation servic
 - `--keep-running`: Keeps containers running after tests complete for manual validation
 - `--element`: Includes Element web client in the test environment
 - `--no-test`: Starts containers and skips running tests (useful for manual testing or debugging)
+- `--observability`: Enables observability services (Grafana, Prometheus, Tempo) for tracing and metrics collection. Always includes Element web client and automatically keeps containers running after tests complete. Use this flag when you want to test observability code to verify that traces and metrics are being collected correctly. This requires manual testing/verification in Grafana and Prometheus, but the script helps set up the environment.
 
 ### Usage Examples
 
 **Basic local testing:**
+
 ```bash
 yarn test:integration
 ```
 
 **Test with pre-built image:**
+
 ```bash
 yarn test:integration --image
 ```
 
 **Test with specific pre-built image:**
+
 ```bash
 yarn test:integration --image rocketchat/rocket.chat:latest
 ```
 
 **Keep services running for manual inspection:**
+
 ```bash
 yarn test:integration --keep-running
 ```
 
 **Run with Element client:**
+
 ```bash
 yarn test:integration --element
 ```
 
 **Start containers only (skip tests):**
+
 ```bash
 yarn test:integration --no-test
 ```
 
 **Start containers with Element and keep them running (skip tests):**
+
 ```bash
 yarn test:integration --keep-running --element --no-test
 ```
 
+**Run with observability enabled:**
+
+```bash
+yarn test:integration --observability
+```
+
 **Combine flags:**
+
 ```bash
 yarn test:integration --image rocketchat/rocket.chat:latest --keep-running --element
 ```
 
 ### Service URLs (when using --keep-running or --no-test)
 
 - **Rocket.Chat**: https://rc1
-- **Synapse**: https://hs1  
+- **Synapse**: https://hs1
 - **MongoDB**: localhost:27017
-- **Element**: https://element (when using --element flag)
+- **Element**: https://element (when using --element flag or --observability flag)
+
+### Observability Services (when using --observability flag)
+
+When the `--observability` flag is enabled, the following services are started for tracing and metrics collection:
+
+- **Grafana**: http://localhost:4001 - Visualization dashboard for traces and metrics
+- **Prometheus**: http://localhost:9090 - Metrics collection and querying
+- **Tempo**: http://localhost:3200 - Distributed tracing backend
+
+The observability services are automatically configured to:
+
+- Collect traces from Rocket.Chat via OpenTelemetry (OTLP) on port 4317
+- Scrape Prometheus metrics from Rocket.Chat on port 9458
+- Display traces and metrics in Grafana with pre-configured datasources
+
+**Use Case**: Use the `--observability` flag when you want to test observability code to see if it's collecting the right information. This requires manual testing and verification - you'll need to interact with the federation services, then check Grafana and Prometheus to verify that traces and metrics are being collected correctly. The script helps set up the environment, but the actual verification must be done manually.
+
+**Note**: The `--observability` flag automatically keeps containers running after tests complete (equivalent to `--keep-running`) and always includes Element web client. You don't need to specify `--element` separately when using `--observability`.

ee/packages/federation-matrix/TRACING_ENTRY_POINTS.md

@@ -0,0 +1,72 @@
+# Federation Tracing Entry Points
+
+This document identifies the entry points where root traces should be created for federation operations.
+
+## Entry Points for Root Traces
+
+### 1. Incoming Messages from Matrix Transactions
+
+**Entry Point:** `PUT /_matrix/federation/v1/send/:txnId`
+
+- **Location:** `ee/packages/federation-matrix/src/api/_matrix/transactions.ts:335`
+- **Flow:**
+  1. HTTP endpoint receives transaction → creates root trace (via tracer middleware)
+  2. Calls `federationSDK.processIncomingTransaction(body)`
+  3. Which calls `eventService.notify()` → `event.service.ts:907`
+  4. Which emits `eventEmitterService.emit('homeserver.matrix.message', ...)` → `event.service.ts:925`
+  5. Which is handled by `federationSDK.eventEmitterService.on('homeserver.matrix.message', ...)` → `message.ts:114`
+
+**Problem:** The `homeserver.matrix.message` event becomes a child of the HTTP request trace, but it should be its own root trace since it represents an independent operation.
+
+**Solution:** When emitting `homeserver.matrix.*` events from `notify()`, they should create root spans by using `ROOT_CONTEXT` instead of the active context.
+
+### 2. Incoming Invites from Matrix Transactions
+
+**Entry Point:** `PUT /_matrix/federation/v1/send/:txnId` (same as messages)
+
+- **Flow:**
+  1. HTTP endpoint → `federationSDK.processIncomingTransaction()`
+  2. → `eventService.notify()` → emits `homeserver.matrix.membership` (invite type)
+  3. → Handled by `member.ts:120` (`handleInvite`)
+
+**Problem:** Same as messages - becomes child of HTTP request trace.
+
+### 3. Incoming Invites from Matrix Invite Endpoint
+
+**Entry Point:** `PUT /_matrix/federation/v2/invite/:roomId/:eventId`
+
+- **Location:** `ee/packages/federation-matrix/src/api/_matrix/invite.ts:135`
+- **Flow:**
+  1. HTTP endpoint → creates root trace (via tracer middleware)
+  2. Calls `federationSDK.processInvite(event, eventId, roomVersion, strippedStateEvents)`
+  3. Which eventually processes the invite and may emit events
+
+**Note:** This endpoint should already be a root trace via the tracer middleware.
+
+### 4. Outgoing Invites (User Calls Invite Endpoint)
+
+**Entry Point:** `POST /v1/rooms.invite`
+
+- **Location:** `apps/meteor/app/api/server/v1/rooms.ts:1083`
+- **Flow:**
+  1. HTTP endpoint → creates root trace (via tracer middleware)
+  2. Calls `FederationMatrix.handleInvite(roomId, userId, action)` OR
+  3. Triggers `beforeAddUserToRoom` hook → `ee/server/hooks/federation/index.ts:105`
+  4. Which calls `FederationMatrix.inviteUsersToRoom(room, [user.username], inviter)` → `FederationMatrix.ts:581`
+  5. Which calls `federationSDK.inviteUserToRoom()`
+
+**Note:** This endpoint should already be a root trace via the tracer middleware.
+
+## Current Problem
+
+When `eventEmitterService.emit('homeserver.matrix.*')` is called from within `notify()`, it's being called from within the context of the HTTP request trace (`PUT /_matrix/federation/v1/send/:txnId`). This causes all `homeserver.matrix.*` events to become children of the HTTP request trace instead of being independent root traces.
+
+## Solution
+
+Modify the `EventEmitterService.emit()` method in the homeserver SDK to:
+
+1. Detect when emitting `homeserver.*` events (incoming Matrix operations)
+2. Use `ROOT_CONTEXT` instead of the active context to create root spans
+3. This ensures each incoming Matrix operation gets its own independent trace
+
+The same should be done for `EventEmitterService.on()` handlers to ensure they also create root spans when handling incoming Matrix events.