add host metrics config example

config_examples/README.md

@@ -23,6 +23,7 @@ Dynatrace distribution of the OpenTelemetry Collector.
 - [Syslog Receiver](syslog.yaml)
 - [Zipkin Receiver](zipkin.yaml)
 - [Redaction Processor](redaction.yaml)
+- [Host Metrics Receiver](host-metrics.yaml)
 
 ## Sending data to Dynatrace
 

config_examples/host-metrics.yaml

@@ -0,0 +1,67 @@
+receivers:
+  hostmetrics:
+    collection_interval: 10s
+    scrapers:
+      paging:
+        metrics:
+          system.paging.utilization:
+            enabled: true
+      cpu:
+        metrics:
+          system.cpu.logical.count:
+            enabled: true
+          system.cpu.physical.count:
+            enabled: true
+          system.cpu.utilization:
+            enabled: true
+      disk:
+      filesystem:
+        metrics:
+          system.filesystem.utilization:
+            enabled: true
+      load:
+      memory:
+        metrics:
+          system.memory.limit:
+            enabled: true
+      network:
+      processes:
+      process:
+        mute_process_all_errors: true
+        metrics:
+          process.cpu.utilization:
+            enabled: true
+          process.memory.utilization:
+            enabled: true
+      system:
+
+processors:
+  batch:
+    send_batch_size: 3000
+    send_batch_max_size: 3000
+    timeout: 60s
+  cumulativetodelta:
+  resourcedetection:
+    detectors: ["system"]
+    system:
+      resource_attributes:
+        host.arch:
+          enabled: true
+        host.ip:
+          enabled: true
+        host.mac:
+          enabled: true
+
+exporters:
+  debug:
+#    verbosity: detailed
+  otlphttp:
+    endpoint: "http://localhost:8000"
+    compression: none
+
+service:
+  pipelines:
+    metrics:
+      receivers: [hostmetrics]
+      processors: [resourcedetection, cumulativetodelta]
+      exporters: [otlphttp, debug]

internal/data-ingest-cli/README.md

@@ -18,6 +18,7 @@ making it easier to diagnose and resolve problems related to OTLP data handling.
   - Syslog
   - Fluent
   - Zipkin
+  - Host metrics
 
 - Receive OTLP data via an endpoint: The tool can receive OTLP data through an endpoint and store the received payload in a specified file in OTLP Json format.
 
@@ -37,12 +38,14 @@ go build -o data-ingest
 
 The tool accepts the following input parameters:
 
+- `--send`: Set to true to send data using the CLI.
+- `--receive`: Set to true to receive data using the CLI.
 - `--input-file`: The name of the input file to read data from.
 - `--input-format`: The input format of the ingested data (options: `otlp-json`, `syslog`, `statsd`, `fluent`).
 - `--collector-url`: The URL of the OpenTelemetry collector.
 - `--output-file`: The file in which to store the received data.
 - `--receiver-port`: The port of the OTLP receiver created to act as a sink for the collector.
-- `--receiver-type`: The type of receiver created to act as a sink for the collector (options: `http`, `grpc`). Please not, that when using the `http` option with Collector's `otlphttp exporter`, you need to disable the compression on the exporter, as no decompression is supported.
+- `--receiver-type`: The type of receiver created to act as a sink for the collector (options: `http`, `grpc`). Please note, that when using the `http` option with Collector's `otlphttp exporter`, you need to disable the compression on the exporter, as no decompression is supported.
 - `--statsd-protocol`: Statsd protocol to send metrics (options: 'udp', 'udp4', 'udp6', 'tcp', 'tcp4', 'tcp6', 'unixgram').
 - `--syslog-transport`: Syslog network transport (options: 'udp', 'tcp')
 - `--zipkin-version`: The version of zipkin traces (options: `v1`, `v2`). Default `v2`.
@@ -52,25 +55,25 @@ The tool accepts the following input parameters:
 1. Send OTLP JSON data to a collector:
 
 ```shell
-./data-ingest --input-format otlp-json --input-file $(pwd)/commands/otlpjson/testdata/traces.json  --otlp-signal-type traces --collector-url localhost:4317 --output-file received_traces.json --receiver-port 4319 --receiver-type http
+./data-ingest --send --input-format otlp-json --input-file $(pwd)/commands/otlpjson/testdata/traces.json  --otlp-signal-type traces --collector-url localhost:4317 --output-file received_traces.json --receiver-port 4319 --receiver-type http
 ```
 
 1. Send statsd data to a collector:
 
 ```shell
-./data-ingest --input-format statsd --input-file $(pwd)/commands/statsd/testdata/metrics.txt --collector-url localhost:8125 --output-file received_metrics.json --receiver-port 4319 --statsd-protocol udp --otlp-signal-type metrics --receiver-type http
+./data-ingest --send --input-format statsd --input-file $(pwd)/commands/statsd/testdata/metrics.txt --collector-url localhost:8125 --output-file received_metrics.json --receiver-port 4319 --statsd-protocol udp --otlp-signal-type metrics --receiver-type http
 ```
 
 1. Send Syslog data to a collector:
 
 ```shell
-./data-ingest --input-format syslog --input-file $(pwd)/commands/syslog/testdata/rfc6587-non-transparent-framing --collector-url localhost:54526
+./data-ingest --send --input-format syslog --input-file $(pwd)/commands/syslog/testdata/rfc6587-non-transparent-framing --collector-url localhost:54526
 ```
 
 1. Send Fluent data to a collector:
 
 ```shell
-./data-ingest --input-format fluent --input-file $(pwd)/commands/fluent/testdata/msg.json --collector-url localhost:8006
+./data-ingest --send --input-format fluent --input-file $(pwd)/commands/fluent/testdata/msg.json --collector-url localhost:8006
 ```
 
 1. Send Zipkin data to a collector:

internal/data-ingest-cli/commands/fluent/command.go

@@ -4,18 +4,21 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+
 	"github.com/Dynatrace/dynatrace-otel-collector/internal/data-ingest-cli/receiver"
 	otlpreceiver "github.com/Dynatrace/dynatrace-otel-collector/internal/data-ingest-cli/receiver/otlp"
 	"github.com/Dynatrace/dynatrace-otel-collector/internal/data-ingest-cli/receiver/otlphttp"
 	"github.com/Dynatrace/dynatrace-otel-collector/internal/data-ingest-cli/sender/fluent"
-	"net/url"
-	"os"
-	"strconv"
 )
 
 var errMissingFluentProperties = fmt.Errorf("test data must be a json object containing a 'tag' and 'message' property")
 
 type Config struct {
+	SendData     bool
+	ReceiveData  bool
 	InputFile    string
 	CollectorURL string
 	Transport    string
@@ -35,7 +38,7 @@ func New(cfg Config) (*Cmd, error) {
 		cfg: cfg,
 	}
 
-	if cfg.ReceiverPort > 0 && cfg.OutputFile != "" {
+	if cfg.ReceiveData && cfg.ReceiverPort > 0 && cfg.OutputFile != "" {
 		switch cfg.ReceiverType {
 		case "grpc":
 			c.receiver = otlpreceiver.NewOTLPReceiver(otlpreceiver.Config{
@@ -52,19 +55,21 @@ func New(cfg Config) (*Cmd, error) {
 		}
 	}
 
-	collectorURL, err := url.Parse(cfg.CollectorURL)
-	if err != nil {
-		return nil, err
-	}
-	port, err := strconv.Atoi(collectorURL.Port())
-	if err != nil {
-		return nil, err
-	}
-	sender, err := fluent.New(collectorURL.Hostname(), port)
-	if err != nil {
-		return nil, err
+	if cfg.SendData {
+		collectorURL, err := url.Parse(cfg.CollectorURL)
+		if err != nil {
+			return nil, err
+		}
+		port, err := strconv.Atoi(collectorURL.Port())
+		if err != nil {
+			return nil, err
+		}
+		sender, err := fluent.New(collectorURL.Hostname(), port)
+		if err != nil {
+			return nil, err
+		}
+		c.sender = sender
 	}
-	c.sender = sender
 
 	return c, nil
 }
@@ -80,6 +85,9 @@ func (c *Cmd) Do(_ context.Context) error {
 }
 
 func (c *Cmd) sendLogs() error {
+	if c.sender == nil {
+		return nil
+	}
 	fileContent, err := os.ReadFile(c.cfg.InputFile)
 	if err != nil {
 		return err

internal/data-ingest-cli/commands/otlpjson/command.go

@@ -15,6 +15,8 @@ import (
 )
 
 type Config struct {
+	SendData     bool
+	ReceiveData  bool
 	InputFile    string
 	CollectorURL string
 	SignalType   string
@@ -36,14 +38,15 @@ func New(p Config) (*Cmd, error) {
 		inputFile:  p.InputFile,
 	}
 
-	sender, err := otlp.New(p.CollectorURL)
-	if err != nil {
-		return nil, err
+	if p.SendData {
+		sender, err := otlp.New(p.CollectorURL)
+		if err != nil {
+			return nil, err
+		}
+		c.sender = sender
 	}
 
-	c.sender = sender
-
-	if p.ReceiverPort > 0 && p.OutputFile != "" {
+	if p.ReceiveData && p.ReceiverPort > 0 && p.OutputFile != "" {
 		switch p.ReceiverType {
 		case "grpc":
 			c.receiver = otlpreceiver.NewOTLPReceiver(otlpreceiver.Config{
@@ -83,6 +86,9 @@ func (c *Cmd) Do(ctx context.Context) error {
 }
 
 func (c *Cmd) sendMetrics(ctx context.Context) error {
+	if c.sender == nil {
+		return nil
+	}
 	fileContent, err := os.ReadFile(c.inputFile)
 	if err != nil {
 		return err
@@ -96,6 +102,9 @@ func (c *Cmd) sendMetrics(ctx context.Context) error {
 }
 
 func (c *Cmd) sendLogs(ctx context.Context) error {
+	if c.sender == nil {
+		return nil
+	}
 	fileContent, err := os.ReadFile(c.inputFile)
 	if err != nil {
 		return err
@@ -109,6 +118,9 @@ func (c *Cmd) sendLogs(ctx context.Context) error {
 }
 
 func (c *Cmd) sendTraces(ctx context.Context) error {
+	if c.sender == nil {
+		return nil
+	}
 	fileContent, err := os.ReadFile(c.inputFile)
 	if err != nil {
 		return err

internal/data-ingest-cli/commands/otlpjson/testdata/metrics.json

@@ -1 +1,26 @@
-{"resourceMetrics":[{"resource":{},"scopeMetrics":[{"scope":{},"metrics":[{"name":"gen","gauge":{"dataPoints":[{"timeUnixNano":"1737962958727640000","asInt":"0"}]}}]}],"schemaUrl":"https://opentelemetry.io/schemas/1.13.0"}]}
+{
+  "resourceMetrics": [
+    {
+      "resource": {},
+      "scopeMetrics": [
+        {
+          "scope": {},
+          "metrics": [
+            {
+              "name": "gen",
+              "gauge": {
+                "dataPoints": [
+                  {
+                    "timeUnixNano": "1737962958727640000",
+                    "asInt": "0"
+                  }
+                ]
+              }
+            }
+          ]
+        }
+      ],
+      "schemaUrl": "https://opentelemetry.io/schemas/1.13.0"
+    }
+  ]
+}

internal/data-ingest-cli/commands/statsd/command.go

@@ -12,6 +12,8 @@ import (
 )
 
 type Config struct {
+	SendData     bool
+	ReceiveData  bool
 	InputFile    string
 	CollectorURL string
 	SignalType   string
@@ -38,14 +40,15 @@ func New(p Config) (*Cmd, error) {
 		inputFile:  p.InputFile,
 	}
 
-	sender, err := statsd.New(p.CollectorURL, p.Protocol)
-	if err != nil {
-		return nil, err
+	if p.SendData {
+		sender, err := statsd.New(p.CollectorURL, p.Protocol)
+		if err != nil {
+			return nil, err
+		}
+		c.sender = sender
 	}
 
-	c.sender = sender
-
-	if p.ReceiverPort > 0 && p.OutputFile != "" {
+	if p.ReceiveData && p.ReceiverPort > 0 && p.OutputFile != "" {
 		switch p.ReceiverType {
 		case "grpc":
 			c.receiver = otlpreceiver.NewOTLPReceiver(otlpreceiver.Config{
@@ -77,6 +80,9 @@ func (c *Cmd) Do(ctx context.Context) error {
 }
 
 func (c *Cmd) sendMetrics(ctx context.Context) error {
+	if c.sender == nil {
+		return nil
+	}
 	fileContent, err := os.ReadFile(c.inputFile)
 	if err != nil {
 		return err

internal/data-ingest-cli/commands/syslog/command.go

@@ -12,6 +12,8 @@ import (
 )
 
 type Config struct {
+	SendData     bool
+	ReceiveData  bool
 	InputFile    string
 	CollectorURL string
 	Transport    string
@@ -23,6 +25,7 @@ type Config struct {
 type Cmd struct {
 	cfg      Config
 	receiver receiver.Receiver
+	sender   *syslog.Sender
 }
 
 func New(cfg Config) (*Cmd, error) {
@@ -34,7 +37,18 @@ func New(cfg Config) (*Cmd, error) {
 		cfg: cfg,
 	}
 
-	if cfg.ReceiverPort > 0 && cfg.OutputFile != "" {
+	if cfg.SendData {
+		sender, err := syslog.Connect(context.Background(), &syslog.Config{
+			Endpoint:  c.cfg.CollectorURL,
+			Transport: c.cfg.Transport,
+		})
+		if err != nil {
+			return nil, err
+		}
+		c.sender = sender
+	}
+
+	if cfg.ReceiveData && cfg.ReceiverPort > 0 && cfg.OutputFile != "" {
 		switch cfg.ReceiverType {
 		case "grpc":
 			c.receiver = otlpreceiver.NewOTLPReceiver(otlpreceiver.Config{
@@ -65,18 +79,13 @@ func (c *Cmd) Do(ctx context.Context) error {
 }
 
 func (c *Cmd) sendLogs(ctx context.Context) error {
-	fileContent, err := os.ReadFile(c.cfg.InputFile)
-	if err != nil {
-		return err
+	if c.sender == nil {
+		return nil
 	}
-
-	sender, err := syslog.Connect(ctx, &syslog.Config{
-		Endpoint:  c.cfg.CollectorURL,
-		Transport: c.cfg.Transport,
-	})
+	fileContent, err := os.ReadFile(c.cfg.InputFile)
 	if err != nil {
 		return err
 	}
 
-	return sender.Write(ctx, string(fileContent))
+	return c.sender.Write(ctx, string(fileContent))
 }