Skip to content

Commit

Permalink
misc: finalized instrumentation setup
Browse files Browse the repository at this point in the history
  • Loading branch information
sheensantoscapadngan committed Nov 19, 2024
1 parent 7c84adc commit 0693f81
Show file tree
Hide file tree
Showing 9 changed files with 563 additions and 956 deletions.
4 changes: 3 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ up-dev:
up-dev-ldap:
docker compose -f docker-compose.dev.yml --profile ldap up --build

up-dev-metrics:
docker compose -f docker-compose.dev.yml --profile metrics up --build

up-prod:
docker-compose -f docker-compose.prod.yml up --build

Expand All @@ -27,4 +30,3 @@ reviewable-api:
npm run type:check

reviewable: reviewable-ui reviewable-api

1,423 changes: 497 additions & 926 deletions backend/package-lock.json

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,11 @@
"@octokit/plugin-retry": "^5.0.5",
"@octokit/rest": "^20.0.2",
"@octokit/webhooks-types": "^7.3.1",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/auto-instrumentations-node": "^0.46.1",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1",
"@opentelemetry/exporter-prometheus": "^0.51.1",
"@opentelemetry/instrumentation": "^0.51.1",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.53.0",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.55.0",
"@opentelemetry/exporter-prometheus": "^0.55.0",
"@opentelemetry/instrumentation": "^0.55.0",
"@opentelemetry/resources": "^1.28.0",
"@opentelemetry/sdk-metrics": "^1.28.0",
"@opentelemetry/semantic-conventions": "^1.27.0",
Expand Down
6 changes: 3 additions & 3 deletions backend/src/lib/config/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,11 +212,11 @@ let envCfg: Readonly<z.infer<typeof envSchema>>;

export const getConfig = () => envCfg;
// cannot import singleton logger directly as it needs config to load various transport
export const initEnvConfig = (logger: Logger) => {
export const initEnvConfig = (logger?: Logger) => {
const parsedEnv = envSchema.safeParse(process.env);
if (!parsedEnv.success) {
logger.error("Invalid environment variables. Check the error below");
logger.error(parsedEnv.error.issues);
(logger ?? console).error("Invalid environment variables. Check the error below");
(logger ?? console).error(parsedEnv.error.issues);
process.exit(-1);
}

Expand Down
30 changes: 26 additions & 4 deletions backend/src/lib/telemetry/instrumentation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,14 @@ import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
import { registerInstrumentations } from "@opentelemetry/instrumentation";
import { Resource } from "@opentelemetry/resources";
import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
import dotenv from "dotenv";

export const initTelemetryInstrumentation = async ({
import { initEnvConfig } from "../config/env";

dotenv.config();

const initTelemetryInstrumentation = ({
exportType,
otlpURL,
otlpUser,
Expand All @@ -24,8 +29,8 @@ export const initTelemetryInstrumentation = async ({

const resource = Resource.default().merge(
new Resource({
[SEMRESATTRS_SERVICE_NAME]: "infisical-server",
[SEMRESATTRS_SERVICE_VERSION]: "0.1.0"
[ATTR_SERVICE_NAME]: "infisical-server",
[ATTR_SERVICE_VERSION]: "0.1.0"
})
);

Expand Down Expand Up @@ -67,3 +72,20 @@ export const initTelemetryInstrumentation = async ({
instrumentations: [getNodeAutoInstrumentations()]
});
};

const setupTelemetry = () => {
const appCfg = initEnvConfig();

if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
console.log("Initializing telemetry instrumentation");
initTelemetryInstrumentation({
otlpURL: appCfg.OTEL_EXPORT_OTLP_ENDPOINT,
otlpUser: appCfg.OTEL_COLLECTOR_BASIC_AUTH_USERNAME,
otlpPassword: appCfg.OTEL_COLLECTOR_BASIC_AUTH_PASSWORD,
otlpPushInterval: appCfg.OTEL_OTLP_PUSH_INTERVAL,
exportType: appCfg.OTEL_EXPORT_TYPE
});
}
};

void setupTelemetry();
13 changes: 2 additions & 11 deletions backend/src/main.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import "./lib/telemetry/instrumentation";

import dotenv from "dotenv";
import path from "path";

Expand All @@ -8,7 +10,6 @@ import { keyStoreFactory } from "./keystore/keystore";
import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
import { isMigrationMode } from "./lib/fn";
import { initLogger } from "./lib/logger";
import { initTelemetryInstrumentation } from "./lib/telemetry/instrumentation";
import { queueServiceFactory } from "./queue";
import { main } from "./server/app";
import { bootstrapCheck } from "./server/boot-strap-check";
Expand All @@ -20,16 +21,6 @@ const run = async () => {
const logger = await initLogger();
const appCfg = initEnvConfig(logger);

if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
await initTelemetryInstrumentation({
otlpURL: appCfg.OTEL_EXPORT_OTLP_ENDPOINT,
otlpUser: appCfg.OTEL_COLLECTOR_BASIC_AUTH_USERNAME,
otlpPassword: appCfg.OTEL_COLLECTOR_BASIC_AUTH_PASSWORD,
otlpPushInterval: appCfg.OTEL_OTLP_PUSH_INTERVAL,
exportType: appCfg.OTEL_EXPORT_TYPE
});
}

const db = initDbConnection({
dbConnectionUri: appCfg.DB_CONNECTION_URI,
dbRootCert: appCfg.DB_ROOT_CERT,
Expand Down
2 changes: 1 addition & 1 deletion backend/src/server/plugins/api-metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import fp from "fastify-plugin";

export const apiMetrics = fp(async (fastify) => {
const apiMeter = opentelemetry.metrics.getMeter("API");
const latencyHistogram = apiMeter.createHistogram("API latency", {
const latencyHistogram = apiMeter.createHistogram("API_latency", {
unit: "ms"
});

Expand Down
24 changes: 24 additions & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ services:
- .env
ports:
- 4000:4000
- 9464:9464 # for OTEL collection of Prometheus metrics
environment:
- NODE_ENV=development
- DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable
Expand All @@ -95,6 +96,28 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"

prometheus:
image: prom/prometheus
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
profiles: [metrics]

grafana:
image: grafana/grafana
container_name: grafana
restart: unless-stopped
environment:
- GF_LOG_LEVEL=debug
ports:
- "3005:3000"
volumes:
- "grafana_storage:/var/lib/grafana"
profiles: [metrics]

frontend:
container_name: infisical-dev-frontend
restart: unless-stopped
Expand Down Expand Up @@ -166,3 +189,4 @@ volumes:
driver: local
ldap_data:
ldap_config:
grafana_storage:
7 changes: 2 additions & 5 deletions prometheus.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
scrape_configs:
- job_name: "otel-collector"
- job_name: "metric-collector"
scrape_interval: 30s
static_configs:
- targets: ["otel-collector:8889"]
basic_auth:
username: infisical
password: infisical
- targets: ["backend:9464"]

0 comments on commit 0693f81

Please sign in to comment.