feat: add OpenTelemetry distributed tracing to daemon and Lambdas

packages/daemon/docker-compose.dev.yml

@@ -0,0 +1,27 @@
+services:
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: hathor
+      MYSQL_DATABASE: wallet_service
+      MYSQL_USER: hathor
+      MYSQL_PASSWORD: hathor
+    ports:
+      - "3306:3306"
+    volumes:
+      - daemon-mysql-data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 15s
+
+  jaeger:
+    image: jaegertracing/jaeger:latest
+    ports:
+      - "16686:16686"
+      - "4318:4318"
+
+volumes:
+  daemon-mysql-data:

packages/daemon/env.development

@@ -0,0 +1,33 @@
+# Fullnode connection (remote)
+# Point to any fullnode with the event WebSocket API enabled.
+# USE_SSL=true for wss:// and https://, false for ws:// and http://
+export FULLNODE_HOST=wallet-service.private-nodes.hathor.network
+export USE_SSL=true
+export NETWORK=mainnet
+export FULLNODE_NETWORK=mainnet
+
+# These are fetched from the fullnode on first connect.
+# Run: yarn dev:fetch-ids
+# Then copy STREAM_ID and FULLNODE_PEER_ID from export-identifiers.sh
+export FULLNODE_PEER_ID=d9ace4305193d76b9d5ec83831d69d31fa7fbcd43b05a448b73269d21c5cc04f
+export STREAM_ID=c373e689-0b37-4df6-accf-829bd5c2013e
+
+# Local MySQL (from docker-compose.dev.yml)
+export DB_ENDPOINT=localhost
+export DB_NAME=wallet_service
+export DB_USER=hathor
+export DB_PASS=hathor
+export DB_PORT=3306
+
+# AWS stubs (not used locally, but required by config validation)
+export NEW_TX_SQS=local-stub
+export PUSH_NOTIFICATION_ENABLED=false
+export WALLET_SERVICE_LAMBDA_ENDPOINT=local-stub
+export STAGE=local
+export ACCOUNT_ID=000000000000
+export ALERT_MANAGER_TOPIC=local-stub
+export ALERT_MANAGER_REGION=us-east-1
+export APPLICATION_NAME=wallet-service-daemon
+
+# OpenTelemetry (optional — remove to disable tracing)
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318

packages/daemon/package.json

@@ -13,6 +13,11 @@
     "lint": "eslint .",
     "build": "tsc -b",
     "start": "node dist/index.js",
+    "dev:up": "docker compose -f docker-compose.dev.yml up -d",
+    "dev:down": "docker compose -f docker-compose.dev.yml down",
+    "dev:destroy": "docker compose -f docker-compose.dev.yml down -v",
+    "dev:migrate": "cd ../.. && DB_ENDPOINT=localhost DB_NAME=wallet_service DB_USER=hathor DB_PASS=hathor DB_PORT=3306 npx sequelize-cli db:migrate && DB_ENDPOINT=localhost DB_NAME=wallet_service DB_USER=hathor DB_PASS=hathor DB_PORT=3306 npx sequelize-cli db:seed:all",
+    "dev:fetch-ids": "FULLNODE_WEBSOCKET_BASEURL=${FULLNODE_HOST} node ../../scripts/fetch-fullnode-ids.js",
     "replay-balance": "yarn dlx ts-node src/scripts/replay-balance.ts",
     "watch": "tsc -w",
     "test_images_up": "docker compose -f ./__tests__/integration/scripts/docker-compose.yml up -d",
@@ -55,6 +60,12 @@
   "dependencies": {
     "@aws-sdk/client-lambda": "3.540.0",
     "@aws-sdk/client-sqs": "3.540.0",
+    "@opentelemetry/api": "1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "0.56.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.57.2",
+    "@opentelemetry/resources": "1.30.1",
+    "@opentelemetry/sdk-node": "0.57.2",
+    "@opentelemetry/sdk-trace-node": "1.30.1",
     "assert": "2.1.0",
     "aws-sdk": "2.1454.0",
     "axios": "1.6.2",

packages/daemon/src/index.ts

@@ -5,6 +5,9 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+// Must be imported before all other modules to patch libraries for auto-instrumentation
+import './tracing';
+
 import { interpret } from 'xstate';
 import { SyncMachine } from './machines';
 import logger from './logger';
@@ -17,7 +20,8 @@ const main = async () => {
   const machine = interpret(SyncMachine);
 
   machine.onTransition((state) => {
-    logger.info(`Transitioned to ${bigIntUtils.JSONBigInt.stringify(state.value)}`);
+    const stateValue = bigIntUtils.JSONBigInt.stringify(state.value);
+    logger.info(`Transitioned to ${stateValue}`);
   });
 
   machine.onDone(() => {

packages/daemon/src/logger.ts

@@ -14,9 +14,10 @@ export default createLogger({
   format: format.combine(
     format.colorize(),
     format.timestamp(),
-    format.printf(({ timestamp, level, message }) => (
-      `${timestamp} [${SERVICE_NAME}][${level}]: ${message}`
-    )),
+    format.printf(({ timestamp, level, message, trace_id, span_id }) => {
+      const traceInfo = trace_id ? ` [trace_id=${trace_id} span_id=${span_id}]` : '';
+      return `${timestamp} [${SERVICE_NAME}][${level}]${traceInfo}: ${message}`;
+    }),
   ),
   transports: [
     new transports.Console(),

packages/daemon/src/machines/SyncMachine.ts

@@ -30,6 +30,7 @@ import {
   handleTokenCreated,
   checkForMissedEvents,
 } from '../services';
+
 import {
   hasNextChange,
   metadataChanged,