From 33b49f44662c08b3386ea9a40215f6eb4dcb2e9e Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 6 Jun 2024 00:42:24 +0800 Subject: [PATCH] misc: finalized config files --- .env.example | 2 +- backend/package-lock.json | 17 ++ backend/package.json | 1 + backend/src/lib/config/env.ts | 2 +- backend/src/lib/telemetry/instrumentation.ts | 33 +-- backend/src/main.ts | 4 +- docker-compose.dev.observable.yml | 201 +++++++++++++++++++ docker-compose.dev.yml | 33 --- otel-collector-config.yaml | 12 +- 9 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 docker-compose.dev.observable.yml diff --git a/.env.example b/.env.example index 23b73efa8b..af63fe5452 100644 --- a/.env.example +++ b/.env.example @@ -64,4 +64,4 @@ CLIENT_SECRET_GITHUB_LOGIN= CLIENT_ID_GITLAB_LOGIN= CLIENT_SECRET_GITLAB_LOGIN= -TELEMETRY_EXPORT_URL= \ No newline at end of file +OTEL_COLLECTOR_OTLP_URL= \ No newline at end of file diff --git a/backend/package-lock.json b/backend/package-lock.json index b51c9fb7ef..b60ffedef3 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -28,6 +28,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/auto-instrumentations-node": "^0.46.1", "@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1", + "@opentelemetry/exporter-prometheus": "^0.51.1", "@opentelemetry/instrumentation": "^0.51.1", "@opentelemetry/resources": "^1.24.1", "@opentelemetry/sdk-metrics": "^1.24.1", @@ -3484,6 +3485,22 @@ "@opentelemetry/api": "^1.3.0" } }, + "node_modules/@opentelemetry/exporter-prometheus": { + "version": "0.51.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.51.1.tgz", + "integrity": "sha512-c8TrTlLm9JJRIHW6MtFv6ESoZRgXBXD/YrTRYylWiyYBOVbYHo1c5Qaw/j/thXDhkmYOYAn4LAhJZpLl5gBFEQ==", + "dependencies": { + "@opentelemetry/core": "1.24.1", + "@opentelemetry/resources": "1.24.1", + "@opentelemetry/sdk-metrics": "1.24.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { "version": "0.51.1", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.51.1.tgz", diff --git a/backend/package.json b/backend/package.json index 59768429aa..b7945356cf 100644 --- a/backend/package.json +++ b/backend/package.json @@ -89,6 +89,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/auto-instrumentations-node": "^0.46.1", "@opentelemetry/exporter-metrics-otlp-proto": "^0.51.1", + "@opentelemetry/exporter-prometheus": "^0.51.1", "@opentelemetry/instrumentation": "^0.51.1", "@opentelemetry/resources": "^1.24.1", "@opentelemetry/sdk-metrics": "^1.24.1", diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 5185d3ea63..a5ffb1db98 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -120,7 +120,7 @@ const envSchema = z .optional(), INFISICAL_CLOUD: zodStrBool.default("false"), MAINTENANCE_MODE: zodStrBool.default("false"), - TELEMETRY_EXPORT_URL: zpStr(z.string().optional()) + OTEL_COLLECTOR_OTLP_URL: zpStr(z.string().optional()) }) .transform((data) => ({ ...data, diff --git a/backend/src/lib/telemetry/instrumentation.ts b/backend/src/lib/telemetry/instrumentation.ts index 308ab73b39..67f295361f 100644 --- a/backend/src/lib/telemetry/instrumentation.ts +++ b/backend/src/lib/telemetry/instrumentation.ts @@ -1,12 +1,13 @@ import opentelemetry from "@opentelemetry/api"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; +import { PrometheusExporter } from "@opentelemetry/exporter-prometheus"; import { registerInstrumentations } from "@opentelemetry/instrumentation"; import { Resource } from "@opentelemetry/resources"; import { AggregationTemporality, MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; -export const initTelemetry = (exportURL: string) => { +export const initTelemetry = async ({ otlpURL }: { otlpURL?: string }) => { const resource = Resource.default().merge( new Resource({ [SEMRESATTRS_SERVICE_NAME]: "infisical-server", @@ -14,22 +15,30 @@ export const initTelemetry = (exportURL: string) => { }) ); - const metricExporter = new OTLPMetricExporter({ - url: `${exportURL}/v1/metrics`, - temporalityPreference: AggregationTemporality.DELTA - }); + const metricReaders = []; + if (otlpURL) { + const otlpExporter = new OTLPMetricExporter({ + url: `${otlpURL}/v1/metrics`, + temporalityPreference: AggregationTemporality.DELTA + }); - const metricReader = new PeriodicExportingMetricReader({ - exporter: metricExporter, - exportIntervalMillis: 30000 - }); + metricReaders.push( + new PeriodicExportingMetricReader({ + exporter: otlpExporter, + exportIntervalMillis: 30000 + }) + ); + } else { + const promExporter = new PrometheusExporter(); + metricReaders.push(promExporter); + } - const myServiceMeterProvider = new MeterProvider({ + const meterProvider = new MeterProvider({ resource, - readers: [metricReader] + readers: metricReaders }); - opentelemetry.metrics.setGlobalMeterProvider(myServiceMeterProvider); + opentelemetry.metrics.setGlobalMeterProvider(meterProvider); registerInstrumentations({ instrumentations: [getNodeAutoInstrumentations()] diff --git a/backend/src/main.ts b/backend/src/main.ts index b8413d43b8..4a1ca74a65 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -15,9 +15,7 @@ const run = async () => { const logger = await initLogger(); const appCfg = initEnvConfig(logger); - if (appCfg.TELEMETRY_EXPORT_URL) { - initTelemetry(appCfg.TELEMETRY_EXPORT_URL); - } + await initTelemetry({ otlpURL: appCfg.OTEL_COLLECTOR_OTLP_URL }); const db = initDbConnection({ dbConnectionUri: appCfg.DB_CONNECTION_URI, diff --git a/docker-compose.dev.observable.yml b/docker-compose.dev.observable.yml new file mode 100644 index 0000000000..f2ddf29b17 --- /dev/null +++ b/docker-compose.dev.observable.yml @@ -0,0 +1,201 @@ +version: "3.9" + +services: + nginx: + container_name: infisical-dev-nginx + image: nginx + restart: always + ports: + - 8080:80 + volumes: + - ./nginx/default.dev.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - backend + - frontend + + db: + image: postgres:14-alpine + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: infisical + POSTGRES_USER: infisical + POSTGRES_DB: infisical + + redis: + image: redis + container_name: infisical-dev-redis + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - 6379:6379 + volumes: + - redis_data:/data + + redis-commander: + container_name: infisical-dev-redis-commander + image: rediscommander/redis-commander + restart: always + depends_on: + - redis + environment: + - REDIS_HOSTS=local:redis:6379 + ports: + - "8085:8081" + + db-test: + profiles: ["test"] + image: postgres:14-alpine + ports: + - "5430:5432" + environment: + POSTGRES_PASSWORD: infisical + POSTGRES_USER: infisical + POSTGRES_DB: infisical-test + + db-migration: + container_name: infisical-db-migration + depends_on: + - db + build: + context: ./backend + dockerfile: Dockerfile.dev + env_file: .env + environment: + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + command: npm run migration:latest + volumes: + - ./backend/src:/app/src + + backend: + container_name: infisical-dev-api + build: + context: ./backend + dockerfile: Dockerfile.dev + depends_on: + db: + condition: service_started + redis: + condition: service_started + db-migration: + condition: service_completed_successfully + env_file: + - .env + ports: + - 4000:4000 + - 9464:9464 # for OTEL collection of Prometheus metrics + environment: + - NODE_ENV=development + - DB_CONNECTION_URI=postgres://infisical:infisical@db/infisical?sslmode=disable + - TELEMETRY_ENABLED=false + volumes: + - ./backend/src:/app/src + extra_hosts: + - "host.docker.internal:host-gateway" + + frontend: + container_name: infisical-dev-frontend + restart: unless-stopped + depends_on: + - backend + build: + context: ./frontend + dockerfile: Dockerfile.dev + volumes: + - ./frontend/src:/app/src/ # mounted whole src to avoid missing reload on new files + - ./frontend/public:/app/public + env_file: .env + environment: + - NEXT_PUBLIC_ENV=development + - INFISICAL_TELEMETRY_ENABLED=false + + pgadmin: + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@example.com + PGADMIN_DEFAULT_PASSWORD: pass + ports: + - 5050:80 + depends_on: + - db + + smtp-server: + container_name: infisical-dev-smtp-server + image: lytrax/mailhog:latest # https://github.com/mailhog/MailHog/issues/353#issuecomment-821137362 + restart: always + logging: + driver: "none" # disable saving logs + ports: + - 1025:1025 # SMTP server + - 8025:8025 # Web UI + + openldap: # note: more advanced configuration is available + image: osixia/openldap:1.5.0 + restart: always + environment: + LDAP_ORGANISATION: Acme + LDAP_DOMAIN: acme.com + LDAP_ADMIN_PASSWORD: admin + ports: + - 389:389 + - 636:636 + volumes: + - ldap_data:/var/lib/ldap + - ldap_config:/etc/ldap/slapd.d + profiles: [ldap] + + phpldapadmin: # username: cn=admin,dc=acme,dc=com, pass is admin + image: osixia/phpldapadmin:latest + restart: always + environment: + - PHPLDAPADMIN_LDAP_HOSTS=openldap + - PHPLDAPADMIN_HTTPS=false + ports: + - 6433:80 + depends_on: + - openldap + profiles: [ldap] + + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the Collector + - 8889:8889 # Prometheus exporter metrics + - 13133:13133 # health_check extension + - 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension + + prometheus: + image: prom/prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + command: + - "--config.file=/etc/prometheus/prometheus.yml" + + grafana: + image: grafana/grafana + container_name: grafana + restart: unless-stopped + environment: + - GF_LOG_LEVEL=debug + ports: + - "3000:3000" + volumes: + - "grafana_storage:/var/lib/grafana" +volumes: + postgres-data: + driver: local + redis_data: + driver: local + ldap_data: + ldap_config: + grafana_storage: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7c742ae6d8..422fe43f3c 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -158,38 +158,6 @@ services: - openldap profiles: [ldap] - otel-collector: - image: otel/opentelemetry-collector-contrib - volumes: - - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml - ports: - - 1888:1888 # pprof extension - - 8888:8888 # Prometheus metrics exposed by the Collector - - 8889:8889 # Prometheus exporter metrics - - 13133:13133 # health_check extension - - 4317:4317 # OTLP gRPC receiver - - 4318:4318 # OTLP http receiver - - 55679:55679 # zpages extension - - prometheus: - image: prom/prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - command: - - "--config.file=/etc/prometheus/prometheus.yml" - - grafana: - image: grafana/grafana - container_name: grafana - restart: unless-stopped - environment: - - GF_LOG_LEVEL=debug - ports: - - "3000:3000" - volumes: - - "grafana_storage:/var/lib/grafana" volumes: postgres-data: driver: local @@ -197,4 +165,3 @@ volumes: driver: local ldap_data: ldap_config: - grafana_storage: diff --git a/otel-collector-config.yaml b/otel-collector-config.yaml index 492aafcd9d..54e810867b 100644 --- a/otel-collector-config.yaml +++ b/otel-collector-config.yaml @@ -3,6 +3,16 @@ receivers: protocols: http: endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: otel-collector + scrape_interval: 30s + static_configs: + - targets: [backend:9464] + metric_relabel_configs: + - action: labeldrop + regex: "service_instance_id|service_name" processors: batch: @@ -20,6 +30,6 @@ service: extensions: [health_check, pprof, zpages] pipelines: metrics: - receivers: [otlp] + receivers: [otlp, prometheus] processors: [batch] exporters: [prometheus]