diff --git a/docs/ingest/telemetry/index.md b/docs/ingest/telemetry/index.md index 1cbf94b6..52a7a158 100644 --- a/docs/ingest/telemetry/index.md +++ b/docs/ingest/telemetry/index.md @@ -60,6 +60,14 @@ scenarios. CrateDB offers corresponding integration adapters. Send metrics with collectd, a system and application metrics collection daemon. :::: +::::{grid-item-card} OpenTelemetry +:link: opentelemetry +:link-type: ref +OpenTelemetry is an open-source observability framework and toolkit designed +to facilitate the export and collection of telemetry data such as traces, +metrics, and logs. +:::: + ::::{grid-item-card} Prometheus :link: prometheus :link-type: ref diff --git a/docs/integrate/opentelemetry/collector/compose.yaml b/docs/integrate/opentelemetry/collector/compose.yaml new file mode 100644 index 00000000..3b03cd4f --- /dev/null +++ b/docs/integrate/opentelemetry/collector/compose.yaml @@ -0,0 +1,43 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + cratedb-ddl: + image: docker.io/crate/crate:latest + command: sh -c "crash --hosts http://cratedb:4200/ -c 'SELECT 1'; crash --hosts http://cratedb:4200/ < /var/ddl.sql" + volumes: + - ./ddl.sql:/var/ddl.sql + depends_on: + - cratedb + + cratedb-prometheus-adapter: + image: ghcr.io/crate/cratedb-prometheus-adapter:0.5.8 + command: -config.file /etc/cratedb-prometheus-adapter.yaml + ports: + - "9268:9268/tcp" + volumes: + - ./cratedb-prometheus-adapter.yaml:/etc/cratedb-prometheus-adapter.yaml + depends_on: + - cratedb-ddl + + otelcol: + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.135.0 + ports: + - "2003:2003/tcp" # Carbon plain text protocol based on TCP + - "4317:4317/tcp" # OTLP gRPC + - "4318:4318/tcp" # OTLP HTTP + volumes: + - ./otelcol.yaml:/etc/otelcol-contrib/config.yaml + depends_on: + - cratedb-prometheus-adapter diff --git a/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml b/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml new file mode 100644 index 00000000..05f4c4b0 --- /dev/null +++ b/docs/integrate/opentelemetry/collector/cratedb-prometheus-adapter.yaml @@ -0,0 +1,20 @@ +cratedb_endpoints: +- host: "cratedb" # Host to connect to (default: "localhost"). + port: 5432 # Port to connect to (default: 5432). + user: "crate" # Username to use (default: "crate") + password: "" # Password to use (default: ""). + schema: "testdrive" # Schema to use (default: ""). + max_connections: 0 # The maximum number of concurrent connections (default: runtime.NumCPU()). + # It will get forwarded to pgx's `pool_max_conns`, and determines + # the maximum number of connections in the connection pool for + # both connection pools (read and write). + read_pool_size_max: 0 # Configure the maximum pool size for read operations individually. + # (default: runtime.NumCPU()) + write_pool_size_max: 0 # Configure the maximum pool size for write operations individually. + # (default: runtime.NumCPU()) + connect_timeout: 10 # TCP connect timeout (seconds) (default: 10). + # It has the same meaning as libpq's `connect_timeout`. + read_timeout: 5 # Query context timeout for read queries (seconds) (default: 5). + write_timeout: 5 # Query context timeout for write queries (seconds) (default: 5). + enable_tls: false # Whether to connect using TLS (default: false). + allow_insecure_tls: false # Whether to allow insecure / invalid TLS certificates (default: false). diff --git a/docs/integrate/opentelemetry/collector/ddl.sql b/docs/integrate/opentelemetry/collector/ddl.sql new file mode 100644 index 00000000..af738a6a --- /dev/null +++ b/docs/integrate/opentelemetry/collector/ddl.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS "testdrive"."metrics" ( + "timestamp" TIMESTAMP, + "labels_hash" TEXT, + "labels" OBJECT(DYNAMIC), + "value" DOUBLE, + "valueRaw" LONG, + "day__generated" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"), + PRIMARY KEY ("timestamp", "labels_hash", "day__generated") +) +PARTITIONED BY ("day__generated") +WITH ("number_of_replicas" = 0); diff --git a/docs/integrate/opentelemetry/collector/example.py b/docs/integrate/opentelemetry/collector/example.py new file mode 100644 index 00000000..1fe51aa4 --- /dev/null +++ b/docs/integrate/opentelemetry/collector/example.py @@ -0,0 +1,16 @@ +# OpenTelemetry demo application. +from opentelemetry import metrics + + +def main(): + + meter = metrics.get_meter("testdrive.meter.name") + temperature = meter.create_gauge("temperature") + humidity = meter.create_gauge("humidity") + + temperature.set(42.42) + humidity.set(84.84) + + +if __name__ == "__main__": + main() diff --git a/docs/integrate/opentelemetry/collector/otelcol.yaml b/docs/integrate/opentelemetry/collector/otelcol.yaml new file mode 100644 index 00000000..5d899280 --- /dev/null +++ b/docs/integrate/opentelemetry/collector/otelcol.yaml @@ -0,0 +1,53 @@ +# OpenTelemetry Collector configuration file +# +# https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/config.yaml +# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter +# +# To limit exposure to denial-of-service attacks, change the host in endpoints below from 0.0.0.0 to a specific network interface. +# See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + carbon: + endpoint: 0.0.0.0:2003 + transport: tcp + parser: + type: plaintext + config: + +processors: + batch: + +exporters: + prometheusremotewrite: + endpoint: "http://cratedb-prometheus-adapter:9268/write" + remote_write_queue: + enabled: false + external_labels: + subsystem: "otel-testdrive" + debug: + verbosity: detailed + +service: + + pipelines: + + metrics: + receivers: [otlp, carbon] + processors: [batch] + exporters: [debug, prometheusremotewrite] + + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/docs/integrate/opentelemetry/collector/usage.md b/docs/integrate/opentelemetry/collector/usage.md new file mode 100644 index 00000000..8e7be3df --- /dev/null +++ b/docs/integrate/opentelemetry/collector/usage.md @@ -0,0 +1,125 @@ +(opentelemetry-otelcol-usage)= +# Connect the OpenTelemetry Collector to CrateDB + +Configure the [OpenTelemetry Collector], its built-in [Prometheus Remote Write Exporter], +and the [CrateDB Prometheus Adapter] to receive [OpenTelemetry] [metrics] and store them +into CrateDB. + +## Prerequisites + +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. + +### Commands + +Prepare shortcut for {ref}`crate-crash:index` command. + +::::{tab-set} +:sync-group: os + +:::{tab-item} Linux and macOS +:sync: unix +Add these settings to your shell profile (`~/.profile`) to make them persistent. +```shell +alias crash="docker compose exec -it cratedb crash" +alias nc="docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23" +``` +::: +:::{tab-item} Windows PowerShell +:sync: powershell +Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. +```powershell +function crash { docker compose exec -it cratedb crash @args } +function nc { docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 @args } +``` +::: +:::{tab-item} Windows Command +:sync: dos +```shell +doskey crash=docker compose exec -it cratedb crash $* +doskey nc=docker run --rm -i --network=cratedb-demo docker.io/toolbelt/netcat:2025-08-23 $* +REM Note: doskey macros reset each session. To persist, configure an AutoRun command +REM pointing to a macro file, or re-run these in your shell startup script. +``` +::: + +:::: + +### Services + +Save {download}`compose.yaml`, {download}`cratedb-prometheus-adapter.yaml`, +{download}`otelcol.yaml` and {download}`ddl.sql` to your machine, then start +services using Docker Compose or Podman Compose. + +```shell +docker compose up +``` + +## Submit data + +### Use netcat + +Use [netcat] to submit metrics using the [Carbon plaintext protocol]. +```shell +printf "temperature;job=app 42.42 1758486061\nhumidity;job=app 84.84 1758486061" | nc -c otelcol 2003 +``` + +### Use Python + +To submit metrics using the OpenTelemetry Python SDK, download the example file +{download}`example.py` to your machine and choose one of these approaches: + +**Option 1: Using uv (recommended)** +```shell +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +export OTEL_SERVICE_NAME=app +uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python example.py +``` + +**Option 2: Using pip** +Install dependencies: +```shell +pip install opentelemetry-distro opentelemetry-exporter-otlp +``` +Then run the example: +```shell +opentelemetry-instrument --service_name=app python example.py +``` + +:::{literalinclude} example.py +::: + +### Use any language + +Use any of the available [OpenTelemetry language APIs & SDKs] for C++, C#/.NET, +Erlang/Elixir, Go, Java, JavaScript, PHP, Python, Ruby, Rust, or Swift. + +## Explore data + +CrateDB stores the metrics in the designated table, ready for inspection and analysis. +```shell +crash --hosts "http://crate:crate@localhost:4200/" -c "SELECT * FROM testdrive.metrics ORDER BY timestamp LIMIT 5;" +``` +```psql ++---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+ +| timestamp | labels_hash | labels | value | valueRaw | day__generated | ++---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+ +| 1758480857158 | 64614d7f1ef80933 | {"__name__": "target_info", "job": "app", "subsystem": "otel-testdrive", "telemetry_auto_version": "0.58b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.37.0"} | 1.0 | 4607182418800017408 | 1758412800000 | +| 1758480857158 | 7c6f57205e58af4c | {"__name__": "temperature", "job": "app", "subsystem": "otel-testdrive"} | 42.42 | 4631166901565532406 | 1758412800000 | +| 1758480857158 | 3fce270356467381 | {"__name__": "humidity", "job": "app", "subsystem": "otel-testdrive"} | 84.84 | 4635670501192902902 | 1758412800000 | ++---------------+------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------+----------------+ +SELECT 3 rows in set (0.005 sec) +``` + + +[Carbon plaintext protocol]: https://graphite.readthedocs.io/en/latest/feeding-carbon.html +[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter +[metrics]: https://opentelemetry.io/docs/concepts/signals/metrics/ +[netcat]: https://en.wikipedia.org/wiki/Netcat +[OpenTelemetry]: https://opentelemetry.io/docs/what-is-opentelemetry/ +[OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ +[OpenTelemetry language APIs & SDKs]: https://opentelemetry.io/docs/languages/ +[Prometheus Remote Write Exporter]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter +[uv]: https://docs.astral.sh/uv/ diff --git a/docs/integrate/opentelemetry/index.md b/docs/integrate/opentelemetry/index.md new file mode 100644 index 00000000..e1bc2559 --- /dev/null +++ b/docs/integrate/opentelemetry/index.md @@ -0,0 +1,82 @@ +(opentelemetry)= +# OpenTelemetry + +```{div} .float-right +[![OpenTelemetry logo](https://opentelemetry.io/img/logos/opentelemetry-horizontal-color.svg){height=100px loading=lazy}][OpenTelemetry] +``` +```{div} .clearfix +``` + +:::{rubric} About +::: + +[OpenTelemetry] (OTel) is an open-source observability framework and toolkit +designed to facilitate the export and collection of telemetry data such as +[traces], [metrics], and [logs]. + +OpenTelemetry provides a unified framework and the APIs/SDKs to instrument +applications, allowing for the use of a single standard across different +observability tools. + +The [OpenTelemetry Collector] and its [Prometheus Remote Write Exporter] can +be used to submit and store [metrics] data into CrateDB. Alternatively, you +can use [Telegraf]. + +:::{rubric} Synopsis +::: + +Configure OpenTelemetry Collector to send metrics data to the [CrateDB Prometheus Adapter]. + +:::{literalinclude} collector/otelcol.yaml +:lines: 26-34 +::: +:::{literalinclude} collector/otelcol.yaml +:lines: 38-43 +::: + +Configure Telegraf to store OpenTelemetry metrics data into CrateDB. + +:::{literalinclude} telegraf/telegraf.conf +:lines: 1-6 +::: +:::{literalinclude} telegraf/telegraf.conf +:lines: 27-33 +::: + + +:::{rubric} Learn +::: + +::::{grid} + +:::{grid-item-card} Guide: Use OTel Collector and CrateDB +:link: opentelemetry-otelcol-usage +:link-type: ref +How to configure OpenTelemetry Collector to submit metrics to CrateDB. +::: + +:::{grid-item-card} Guide: Use Telegraf and CrateDB +:link: opentelemetry-telegraf-usage +:link-type: ref +How to configure Telegraf to submit OpenTelemetry metrics to CrateDB. +::: + +:::: + + +:::{toctree} +:maxdepth: 1 +:hidden: +Collector Usage +Telegraf Usage +::: + + +[CrateDB Prometheus Adapter]: https://github.com/crate/cratedb-prometheus-adapter +[logs]: https://opentelemetry.io/docs/concepts/signals/logs/ +[metrics]: https://opentelemetry.io/docs/concepts/signals/metrics/ +[OpenTelemetry]: https://opentelemetry.io/docs/what-is-opentelemetry/ +[OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ +[Prometheus Remote Write Exporter]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/prometheusremotewriteexporter +[Telegraf]: https://www.influxdata.com/time-series-platform/telegraf/ +[traces]: https://opentelemetry.io/docs/concepts/signals/traces/ diff --git a/docs/integrate/opentelemetry/telegraf/compose.yaml b/docs/integrate/opentelemetry/telegraf/compose.yaml new file mode 100644 index 00000000..c850b9ce --- /dev/null +++ b/docs/integrate/opentelemetry/telegraf/compose.yaml @@ -0,0 +1,23 @@ +# Service composition file for Docker Compose or Podman Compose + +services: + + cratedb: + image: docker.io/crate/crate:latest + ports: + - "4200:4200/tcp" + - "5432:5432/tcp" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4200/"] + interval: 5s + timeout: 30s + retries: 5 + + telegraf: + image: docker.io/telegraf:latest + ports: + - "4317:4317/tcp" # OTLP gRPC + volumes: + - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro + depends_on: + - cratedb diff --git a/docs/integrate/opentelemetry/telegraf/example.py b/docs/integrate/opentelemetry/telegraf/example.py new file mode 100644 index 00000000..1fe51aa4 --- /dev/null +++ b/docs/integrate/opentelemetry/telegraf/example.py @@ -0,0 +1,16 @@ +# OpenTelemetry demo application. +from opentelemetry import metrics + + +def main(): + + meter = metrics.get_meter("testdrive.meter.name") + temperature = meter.create_gauge("temperature") + humidity = meter.create_gauge("humidity") + + temperature.set(42.42) + humidity.set(84.84) + + +if __name__ == "__main__": + main() diff --git a/docs/integrate/opentelemetry/telegraf/telegraf.conf b/docs/integrate/opentelemetry/telegraf/telegraf.conf new file mode 100644 index 00000000..4ade4637 --- /dev/null +++ b/docs/integrate/opentelemetry/telegraf/telegraf.conf @@ -0,0 +1,45 @@ +# OpenTelemetry Input Plugin +# https://github.com/influxdata/telegraf/blob/release-1.36/plugins/inputs/opentelemetry/README.md +[[inputs.opentelemetry]] + ## Override the default (0.0.0.0:4317) destination OpenTelemetry gRPC service + ## address:port + # service_address = "0.0.0.0:4317" + + ## Override the default (5s) new connection timeout + # timeout = "5s" + + ## Override the default (prometheus-v1) metrics schema. + ## Supports: "prometheus-v1", "prometheus-v2" + ## For more information about the alternatives, read the Prometheus input + ## plugin notes. + # metrics_schema = "prometheus-v1" + + ## Optional TLS Config. + ## For advanced options: https://github.com/influxdata/telegraf/blob/v1.18.3/docs/TLS.md + ## + ## Set one or more allowed client CA certificate file names to + ## enable mutually authenticated TLS connections. + # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] + ## Add service certificate and key. + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + +# CrateDB Output Plugin +# https://github.com/influxdata/telegraf/tree/master/plugins/outputs/cratedb +[[outputs.cratedb]] + ## Connection parameters for accessing the database see + ## https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig + ## for available options + url = "postgres://crate:crate@cratedb/doc?sslmode=disable" + + ## Timeout for all CrateDB queries. + # timeout = "5s" + + ## Name of the table to store metrics in. + table = "metrics" + + ## If true, and the metrics table does not exist, create it automatically. + table_create = true + + ## The character(s) to replace any '.' in an object key with + # key_separator = "_" diff --git a/docs/integrate/opentelemetry/telegraf/usage.md b/docs/integrate/opentelemetry/telegraf/usage.md new file mode 100644 index 00000000..f4401a94 --- /dev/null +++ b/docs/integrate/opentelemetry/telegraf/usage.md @@ -0,0 +1,108 @@ +(opentelemetry-telegraf-usage)= +# Store OpenTelemetry metrics using Telegraf and CrateDB + +This how-to guide walks you through configuring [Telegraf] to receive +[OpenTelemetry] [metrics] and store them into CrateDB. + +## Prerequisites + +Use Docker or Podman to run all components. This approach works consistently +across Linux, macOS, and Windows. +If you use Podman, replace `docker` with `podman` (or enable the podman‑docker +compatibility shim) and run `podman compose up`. + +### Commands + +Prepare shortcut for {ref}`crate-crash:index` command. + +::::{tab-set} +:sync-group: os + +:::{tab-item} Linux and macOS +:sync: unix +Add these settings to your shell profile (`~/.profile`) to make them persistent. +```shell +alias crash="docker compose exec -it cratedb crash" +``` +::: +:::{tab-item} Windows PowerShell +:sync: powershell +Add these settings to your PowerShell profile (`$PROFILE`) to make them persistent. +```powershell +function crash { docker compose exec -it cratedb crash @args } +``` +::: +:::{tab-item} Windows Command +:sync: dos +```shell +doskey crash=docker compose exec -it cratedb crash $* +REM Note: doskey macros reset each session. To persist, configure an AutoRun command +REM pointing to a macro file, or re-run these in your shell startup script. +``` +::: + +:::: + +### Services + +Save {download}`compose.yaml` and {download}`telegraf.conf`, then start +services using Docker Compose or Podman Compose. + +```shell +docker compose up +``` + +## Submit data + +### Use Python + +To submit metrics using the OpenTelemetry Python SDK, download the example file +{download}`example.py` to your machine and choose one of these approaches: + +**Option 1: Using uv (recommended)** +```shell +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +export OTEL_SERVICE_NAME=app +uv run --with=opentelemetry-distro --with=opentelemetry-exporter-otlp opentelemetry-instrument python example.py +``` + +**Option 2: Using pip** +Install dependencies: +```shell +pip install opentelemetry-distro opentelemetry-exporter-otlp +``` +Then run the example: +```shell +opentelemetry-instrument --service_name=app python example.py +``` + +:::{literalinclude} example.py +::: + +### Use any language + +Use any of the available [OpenTelemetry language APIs & SDKs] for C++, C#/.NET, +Erlang/Elixir, Go, Java, JavaScript, PHP, Python, Ruby, Rust, or Swift. + +## Explore data + +CrateDB stores the metrics in the designated table, ready for inspection and analysis. +```shell +crash --hosts "http://crate:crate@localhost:4200/" -c "SELECT * FROM metrics ORDER BY timestamp LIMIT 5;" +``` +```psql ++---------------------+---------------+-------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+---------------+ +| hash_id | timestamp | name | tags | fields | day | ++---------------------+---------------+-------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+---------------+ +| 4549776513022193265 | 1758500094846 | temperature | {"host": "2805bf17ee55", "otel_library_name": "testdrive.meter.name", "service_name": "app", "telemetry_auto_version": "0.58b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.37.0"} | {"gauge": 42.42} | 1758499200000 | +| -926134856403504308 | 1758500094846 | humidity | {"host": "2805bf17ee55", "otel_library_name": "testdrive.meter.name", "service_name": "app", "telemetry_auto_version": "0.58b0", "telemetry_sdk_language": "python", "telemetry_sdk_name": "opentelemetry", "telemetry_sdk_version": "1.37.0"} | {"gauge": 84.84} | 1758499200000 | ++---------------------+---------------+-------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+---------------+ +SELECT 2 rows in set (0.049 sec) +``` + + +[metrics]: https://opentelemetry.io/docs/concepts/signals/metrics/ +[OpenTelemetry]: https://opentelemetry.io/docs/what-is-opentelemetry/ +[OpenTelemetry language APIs & SDKs]: https://opentelemetry.io/docs/languages/ +[Telegraf]: https://www.influxdata.com/time-series-platform/telegraf/ +[uv]: https://docs.astral.sh/uv/