feat(controlplane): add service-level tracing spans to Connect-RPC handlers

controlplane/package.json

@@ -28,6 +28,7 @@
     "lint": "eslint --cache --ext .ts,.mjs,.cjs . && prettier -c src",
     "lint:fix": "eslint --cache --fix --ext .ts,.mjs,.cjs . && pnpm format",
     "format": "prettier -w .",
+    "sentry:spotlight": "spotlight",
     "migrate": "pnpm db:migrate && pnpm ch:migrate",
     "drizzle:up": "drizzle-kit up",
     "ch:down": "dbmate -d \"./clickhouse/migrations\" down",
@@ -103,6 +104,7 @@
     "zod-to-json-schema": "^3.22.4"
   },
   "devDependencies": {
+    "@spotlightjs/spotlight": "^4.10.0",
     "@bufbuild/protobuf": "^1.9.0",
     "@bufbuild/protoc-gen-es": "^1.9.0",
     "@connectrpc/protoc-gen-connect-es": "^1.4.0",

controlplane/src/core/auth-utils.ts

@@ -26,7 +26,7 @@ import { OrganizationGroupRepository } from './repositories/OrganizationGroupRep
 import { DefaultNamespace, NamespaceRepository } from './repositories/NamespaceRepository.js';
 import Keycloak from './services/Keycloak.js';
 import { IPlatformWebhookService } from './webhooks/PlatformWebhookService.js';
-
+import { traced } from './tracing.js';
 export type AuthUtilsOptions = {
   webBaseUrl: string;
   webErrorPath: string;
@@ -711,3 +711,5 @@ export default class AuthUtils {
     return 0;
   }
 }
+
+traced(AuthUtils);

controlplane/src/core/blobstorage/dual.ts

@@ -1,3 +1,4 @@
+import { traced } from '../tracing.js';
 import type { BlobObject, BlobStorage } from './index.js';
 
 /**
@@ -6,6 +7,7 @@ import type { BlobObject, BlobStorage } from './index.js';
  * - Writes and deletes go to both stores concurrently; both must succeed.
  * - Reads try the primary first, falling back to the secondary on failure.
  */
+@traced
 export class DualBlobStorage implements BlobStorage {
   constructor(
     private primary: BlobStorage,

controlplane/src/core/blobstorage/s3.ts

@@ -7,6 +7,7 @@ import {
   PutObjectCommand,
   S3Client,
 } from '@aws-sdk/client-s3';
+import { traced } from '../tracing.js';
 import { BlobNotFoundError, BlobObject, type BlobStorage } from './index.js';
 
 const maxConcurrency = 10; // Maximum number of concurrent operations
@@ -26,6 +27,7 @@ export interface S3BlobStorageConfig {
 /**
  * Stores objects in S3 given an S3Client and a bucket name
  */
+@traced
 export class S3BlobStorage implements BlobStorage {
   private readonly useIndividualDeletes: boolean;
 

controlplane/src/core/build-server.ts

@@ -1,6 +1,7 @@
 import Fastify, { FastifyBaseLogger } from 'fastify';
 import { S3Client } from '@aws-sdk/client-s3';
 import { fastifyConnectPlugin } from '@connectrpc/connect-fastify';
+import * as Sentry from '@sentry/node';
 import { cors, createContextValues } from '@connectrpc/connect';
 import fastifyCors from '@fastify/cors';
 import { pino, stdTimeFunctions, LoggerOptions } from 'pino';
@@ -37,7 +38,13 @@ import { BillingRepository } from './repositories/BillingRepository.js';
 import { BillingService } from './services/BillingService.js';
 import { UserRepository } from './repositories/UserRepository.js';
 import { AIGraphReadmeQueue, createAIGraphReadmeWorker } from './workers/AIGraphReadmeWorker.js';
-import { fastifyLoggerId, createS3ClientConfig, extractS3BucketName, isGoogleCloudStorageUrl } from './util.js';
+import {
+  fastifyLoggerId,
+  sentrySpanId,
+  createS3ClientConfig,
+  extractS3BucketName,
+  isGoogleCloudStorageUrl,
+} from './util.js';
 import { ApiKeyRepository } from './repositories/ApiKeyRepository.js';
 import { createDeleteOrganizationWorker, DeleteOrganizationQueue } from './workers/DeleteOrganizationWorker.js';
 import {
@@ -531,6 +538,16 @@ export default async function build(opts: BuildConfig) {
     keycloakRealm: opts.keycloak.realm,
   });
 
+  // Capture the active Sentry span in preHandler (where OTEL context is still available)
+  // and store it on the request so Connect interceptors can use it as parentSpan.
+  fastify.addHook('preHandler', (req, _reply, done) => {
+    const span = Sentry.getActiveSpan();
+    if (span) {
+      (req.raw as any).__sentrySpan = span;
+    }
+    done();
+  });
+
   // Must be registered after custom fastify routes
   // Because it registers an all-catch route for connect handlers
 
@@ -567,7 +584,16 @@ export default async function build(opts: BuildConfig) {
       cdnBaseUrl: opts.cdnBaseUrl,
     }),
     contextValues(req) {
-      return createContextValues().set<FastifyBaseLogger>({ id: fastifyLoggerId, defaultValue: req.log }, req.log);
+      const values = createContextValues().set<FastifyBaseLogger>(
+        { id: fastifyLoggerId, defaultValue: req.log },
+        req.log,
+      );
+      // Read the parent span captured during the preHandler hook
+      const parentSpan = (req.raw as any).__sentrySpan;
+      if (parentSpan) {
+        values.set({ id: sentrySpanId, defaultValue: undefined }, parentSpan);
+      }
+      return values;
     },
     logLevel: opts.logger.level as pino.LevelWithSilent,
     // Avoid compression for small requests
@@ -578,6 +604,18 @@ export default async function build(opts: BuildConfig) {
     // We go with 32MiB to avoid allocating too much memory for large requests
     writeMaxBytes: 32 * 1024 * 1024,
     acceptCompression: [compressionBrotli, compressionGzip],
+    interceptors: [
+      (next) => (req) => {
+        const parentSpan = req.contextValues?.get({
+          id: sentrySpanId,
+          defaultValue: undefined,
+        });
+        if (parentSpan) {
+          return Sentry.withActiveSpan(parentSpan, () => next(req));
+        }
+        return next(req);
+      },
+    ],
   });
 
   await fastify.register(fastifyGracefulShutdown, {

controlplane/src/core/clickhouse/client/ClickHouseClient.ts

@@ -7,6 +7,7 @@ import pkg from 'stream-json';
 
 import { Observable, Subscriber } from 'rxjs';
 
+import { traced } from '../../tracing.js';
 import { ClickHouseCompressionMethod, ClickHouseDataFormat } from './enums/index.js';
 
 import { ClickHouseClientOptions } from './interfaces/index.js';
@@ -16,6 +17,7 @@ const { Parser } = pkg;
  * ClickHouse Client
  * Most of the code is taken from https://github.com/depyronick/clickhouse-client
  */
+@traced
 export class ClickHouseClient {
   /**
    * ClickHouse Endpoint without path and query

controlplane/src/core/composition/composer.ts

@@ -33,6 +33,7 @@ import { CacheWarmerRepository } from '../repositories/CacheWarmerRepository.js'
 import { NamespaceRepository } from '../repositories/NamespaceRepository.js';
 import { InspectorSchemaChange } from '../services/SchemaUsageTrafficInspector.js';
 import { SchemaCheckChangeAction } from '../../db/models.js';
+import { traced } from '../tracing.js';
 import {
   composeGraphsInWorker,
   DeserializedComposedGraph,
@@ -136,6 +137,7 @@ export type CheckSubgraph = {
   // will be used only for new subgraphs
   labels?: Label[];
 };
+@traced
 export class Composer {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/ApiKeyRepository.ts

@@ -4,10 +4,12 @@ import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import * as schema from '../../db/schema.js';
 import { apiKeyPermissions, apiKeyResources, apiKeys, users } from '../../db/schema.js';
 import { APIKeyDTO } from '../../types/index.js';
+import { traced } from '../tracing.js';
 
 /**
  * Repository for organization related operations.
  */
+@traced
 export class ApiKeyRepository {
   constructor(private db: PostgresJsDatabase<typeof schema>) {}
 

controlplane/src/core/repositories/AuditLogRepository.ts

@@ -2,6 +2,7 @@ import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import { desc, eq, gt, lt, and, sql, count } from 'drizzle-orm';
 import * as schema from '../../db/schema.js';
 import { AuditableType, AuditActorType, AuditLogAction, AuditLogFullAction, AuditTargetType } from '../../db/models.js';
+import { traced } from '../tracing.js';
 
 export type AddAuditLogInput = {
   organizationId: string;
@@ -25,6 +26,7 @@ export type AddAuditLogInput = {
 /**
  * Repository for audit log related operations.
  */
+@traced
 export class AuditLogRepository {
   constructor(private db: PostgresJsDatabase<typeof schema>) {}
 

controlplane/src/core/repositories/BillingRepository.ts

@@ -4,6 +4,7 @@ import type { DB } from '../../db/index.js';
 import { billingPlans, billingSubscriptions, organizationBilling } from '../../db/schema.js';
 import { BillingPlanDTO } from '../../types/index.js';
 import { BillingService } from '../services/BillingService.js';
+import { traced } from '../tracing.js';
 
 export const billingSchema = z.object({
   plans: z.record(
@@ -26,6 +27,7 @@ export const billingSchema = z.object({
 /**
  * BillingRepository for billing related operations.
  */
+@traced
 export class BillingRepository {
   constructor(private db: DB) {}
 

controlplane/src/core/repositories/CacheWarmerRepository.ts

@@ -17,6 +17,7 @@ import { BlobStorage } from '../blobstorage/index.js';
 import { ClickHouseClient } from '../clickhouse/index.js';
 import { S3RouterConfigMetadata } from '../composition/composer.js';
 import { CacheWarmupOperation } from '../../db/models.js';
+import { traced } from '../tracing.js';
 import { getDateRange, isoDateRangeToTimestamps } from './analytics/util.js';
 import { OperationsRepository } from './OperationsRepository.js';
 
@@ -28,6 +29,7 @@ interface ComputeCacheWarmerOperationsProps {
   maxOperationsCount: number;
 }
 
+@traced
 export class CacheWarmerRepository {
   constructor(
     private client: ClickHouseClient,

controlplane/src/core/repositories/ContractRepository.ts

@@ -3,8 +3,10 @@ import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import { FastifyBaseLogger } from 'fastify';
 import * as schema from '../../db/schema.js';
 import { FederatedGraphDTO } from '../../types/index.js';
+import { traced } from '../tracing.js';
 import { FederatedGraphRepository } from './FederatedGraphRepository.js';
 
+@traced
 export class ContractRepository {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/FeatureFlagRepository.ts

@@ -31,6 +31,7 @@ import {
 } from '../../types/index.js';
 import { normalizeLabels } from '../util.js';
 import { RBACEvaluator } from '../services/RBACEvaluator.js';
+import { traced } from '../tracing.js';
 import { FederatedGraphRepository } from './FederatedGraphRepository.js';
 import { SubgraphRepository } from './SubgraphRepository.js';
 import { UserRepository } from './UserRepository.js';
@@ -62,6 +63,7 @@ export type CheckConstituentFeatureSubgraphsResult = {
   featureSubgraphIds: Array<string>;
 };
 
+@traced
 export class FeatureFlagRepository {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/FederatedGraphRepository.ts

@@ -74,6 +74,7 @@ import { checkIfLabelMatchersChanged, normalizeLabelMatchers, normalizeLabels }
 import { unsuccessfulBaseCompositionError } from '../errors/errors.js';
 import { ClickHouseClient } from '../clickhouse/index.js';
 import { RBACEvaluator } from '../services/RBACEvaluator.js';
+import { traced } from '../tracing.js';
 import { ContractRepository } from './ContractRepository.js';
 import { FeatureFlagRepository, SubgraphsToCompose } from './FeatureFlagRepository.js';
 import { GraphCompositionRepository } from './GraphCompositionRepository.js';
@@ -85,6 +86,7 @@ export interface FederatedGraphConfig {
   trafficCheckDays: number;
 }
 
+@traced
 export class FederatedGraphRepository {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/GitHubRepository.ts

@@ -4,7 +4,9 @@ import { eq } from 'drizzle-orm';
 import { CompositionError, GitInfo } from '@wundergraph/cosmo-connect/dist/platform/v1/platform_pb';
 import { PlainMessage } from '@bufbuild/protobuf';
 import * as schema from '../../db/schema.js';
+import { traced } from '../tracing.js';
 
+@traced
 export class GitHubRepository {
   constructor(
     private db: PostgresJsDatabase<typeof schema>,

controlplane/src/core/repositories/GraphCompositionRepository.ts

@@ -12,8 +12,10 @@ import {
 } from '../../db/schema.js';
 import { DateRange, GraphCompositionDTO } from '../../types/index.js';
 import { CompositionSubgraphRecord } from '../composition/composer.js';
+import { traced } from '../tracing.js';
 import { FederatedGraphRepository } from './FederatedGraphRepository.js';
 
+@traced
 export class GraphCompositionRepository {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/NamespaceRepository.ts

@@ -3,9 +3,11 @@ import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import * as schema from '../../db/schema.js';
 import { NamespaceDTO } from '../../types/index.js';
 import { RBACEvaluator } from '../services/RBACEvaluator.js';
+import { traced } from '../tracing.js';
 
 export const DefaultNamespace = 'default';
 
+@traced
 export class NamespaceRepository {
   constructor(
     private db: PostgresJsDatabase<typeof schema>,

controlplane/src/core/repositories/OidcRepository.ts

@@ -2,7 +2,9 @@ import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import { eq } from 'drizzle-orm';
 import * as schema from '../../db/schema.js';
 import { oidcProviders } from '../../db/schema.js';
+import { traced } from '../tracing.js';
 
+@traced
 export class OidcRepository {
   constructor(private db: PostgresJsDatabase<typeof schema>) {}
 

controlplane/src/core/repositories/OperationsRepository.ts

@@ -17,6 +17,7 @@ import {
   SchemaCheckDetailsDTO,
   UpdatedPersistedOperation,
 } from '../../types/index.js';
+import { traced } from '../tracing.js';
 import { SchemaCheckRepository } from './SchemaCheckRepository.js';
 
 export const MAX_MANIFEST_OPERATIONS = 3000;
@@ -42,6 +43,7 @@ type IgnoreAllOverride = {
   hash: string;
 };
 
+@traced
 export class OperationsRepository {
   constructor(
     private db: PostgresJsDatabase<typeof schema>,

controlplane/src/core/repositories/OrganizationGroupRepository.ts

@@ -5,7 +5,9 @@ import { OrganizationGroupDTO } from '../../types/index.js';
 import { OrganizationRole } from '../../db/models.js';
 import { organizationRoleEnum } from '../../db/schema.js';
 import { defaultGroupDescription } from '../test-util.js';
+import { traced } from '../tracing.js';
 
+@traced
 export class OrganizationGroupRepository {
   constructor(private db: PostgresJsDatabase<typeof schema>) {}
 

controlplane/src/core/repositories/OrganizationInvitationRepository.ts

@@ -5,10 +5,12 @@ import { FastifyBaseLogger } from 'fastify';
 import * as schema from '../../db/schema.js';
 import { organizationInvitations, organizations, users } from '../../db/schema.js';
 import { OrganizationDTO, OrganizationInvitationDTO, UserDTO } from '../../types/index.js';
+import { traced } from '../tracing.js';
 import { OrganizationRepository } from './OrganizationRepository.js';
 import { UserRepository } from './UserRepository.js';
 import { OrganizationGroupRepository } from './OrganizationGroupRepository.js';
 
+@traced
 export class OrganizationInvitationRepository {
   constructor(
     private logger: FastifyBaseLogger,

controlplane/src/core/repositories/OrganizationRepository.ts

@@ -40,6 +40,7 @@ import { BlobStorage } from '../blobstorage/index.js';
 import { delayForManualOrgDeletionInDays, delayForOrgAuditLogsDeletionInDays } from '../constants.js';
 import { DeleteOrganizationAuditLogsQueue } from '../workers/DeleteOrganizationAuditLogsWorker.js';
 import { RBACEvaluator } from '../services/RBACEvaluator.js';
+import { traced } from '../tracing.js';
 import { BillingRepository } from './BillingRepository.js';
 import { FederatedGraphRepository } from './FederatedGraphRepository.js';
 import { TargetRepository } from './TargetRepository.js';
@@ -48,6 +49,7 @@ import { OrganizationGroupRepository } from './OrganizationGroupRepository.js';
 /**
  * Repository for organization related operations.
  */
+@traced
 export class OrganizationRepository {
   protected billing: BillingRepository;
 

controlplane/src/core/repositories/PlaygroundScriptsRepository.ts

@@ -1,7 +1,9 @@
 import { and, asc, eq } from 'drizzle-orm';
 import { PostgresJsDatabase } from 'drizzle-orm/postgres-js';
 import * as schema from '../../db/schema.js';
+import { traced } from '../tracing.js';
 
+@traced
 export class PlaygroundScriptsRepository {
   constructor(
     private db: PostgresJsDatabase<typeof schema>,

controlplane/src/core/repositories/PluginRepository.ts

@@ -3,7 +3,9 @@ import { FastifyBaseLogger } from 'fastify';
 import { and, eq, count } from 'drizzle-orm';
 import * as schema from '../../db/schema.js';
 import { subgraphs, targets } from '../../db/schema.js';
+import { traced } from '../tracing.js';
 
+@traced
 export class PluginRepository {
   constructor(
     private db: PostgresJsDatabase<typeof schema>,