diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6829c2fde..19a9f3d55 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,6 +6,7 @@ internal/elasticattr @elastic/obs-ds-intake-services @elastic/ loadgen @elastic/obs-ds-intake-services @elastic/obs-ds-hosted-services @elastic/ingest-otel-data receiver/loadgenreceiver @elastic/obs-ds-intake-services @elastic/obs-ds-hosted-services @elastic/ingest-otel-data receiver/elasticapmintakereceiver @elastic/obs-ds-intake-services @elastic/obs-ds-hosted-services @elastic/ingest-otel-data +receiver/akamaisiemreceiver @elastic/security-service-integrations receiver/entityanalyticsreceiver @elastic/security-service-integrations processor/elasticinframetricsprocessor @elastic/obs-infraobs-integrations @elastic/ingest-otel-data processor/elasticapmprocessor @elastic/obs-ds-intake-services @elastic/obs-ds-hosted-services @elastic/ingest-otel-data diff --git a/.gitignore b/.gitignore index 96ec4094a..454b3e892 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,8 @@ _build/ *.iws *.iml *.ipr + +### Misc ### +.vscode/ +.claude/ +.DS_Store diff --git a/distributions/elastic-components/manifest.yaml b/distributions/elastic-components/manifest.yaml index ebb5ffec2..d0bfd0576 100644 --- a/distributions/elastic-components/manifest.yaml +++ b/distributions/elastic-components/manifest.yaml @@ -40,6 +40,7 @@ receivers: - gomod: github.com/elastic/opentelemetry-collector-components/receiver/loadgenreceiver v0.46.0 - gomod: github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver v0.46.0 - gomod: github.com/elastic/opentelemetry-collector-components/receiver/entityanalyticsreceiver v0.0.0 + - gomod: github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver v0.0.0 processors: - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.151.0 @@ -91,5 +92,6 @@ replaces: - github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver => ../receiver/elasticapmintakereceiver - github.com/elastic/opentelemetry-collector-components/receiver/integrationreceiver => ../receiver/integrationreceiver - github.com/elastic/opentelemetry-collector-components/receiver/entityanalyticsreceiver => ../receiver/entityanalyticsreceiver + - github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver => ../receiver/akamaisiemreceiver - github.com/elastic/opentelemetry-collector-components/processor/elastictraceprocessor => ../processor/elastictraceprocessor - github.com/elastic/opentelemetry-collector-components/internal/elasticattr => ../internal/elasticattr diff --git a/receiver/akamaisiemreceiver/Makefile b/receiver/akamaisiemreceiver/Makefile new file mode 100644 index 000000000..ded7a3609 --- /dev/null +++ b/receiver/akamaisiemreceiver/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/receiver/akamaisiemreceiver/README.md b/receiver/akamaisiemreceiver/README.md new file mode 100644 index 000000000..93f614dc4 --- /dev/null +++ b/receiver/akamaisiemreceiver/README.md @@ -0,0 +1,772 @@ +# Akamai SIEM Receiver + + +| Status | | +| ------------- |-----------| +| Stability | [alpha]: logs | +| Distributions | [] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/elastic/opentelemetry-collector-components?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fakamaisiem%20&label=open&color=orange&logo=opentelemetry)](https://github.com/elastic/opentelemetry-collector-components/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fakamaisiem) [![Closed issues](https://img.shields.io/github/issues-search/elastic/opentelemetry-collector-components?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fakamaisiem%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/elastic/opentelemetry-collector-components/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fakamaisiem) | +| Code coverage | [![codecov](https://codecov.io/github/elastic/opentelemetry-collector-components/graph/main/badge.svg?component=receiver_akamaisiem)](https://app.codecov.io/gh/elastic/opentelemetry-collector-components/tree/main/?components%5B0%5D=receiver_akamaisiem&displayType=list) | + +[alpha]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#alpha + + +Polls the [Akamai SIEM API](https://techdocs.akamai.com/siem-integration/reference/get-configid) for security events and emits them as OTel logs in a shape designed for the [Akamai integration](https://docs.elastic.co/integrations/akamai) ingest pipeline in Elasticsearch. + +Each log record carries the raw Akamai JSON event in a body map under the key `message`, alongside `data_stream.{type,dataset,namespace}` body keys (for Kibana filters) and resource attributes (for ES exporter routing). The scope attribute `elastic.mapping.mode: bodymap` tells the Elasticsearch exporter to serialize the body map fields directly into the indexed document. All ECS enrichment is owned by the integration's ingest pipeline — the receiver does not parse or transform event content. + +## Contents + +- [Architecture](#architecture) + - [Data Flow](#data-flow) + - [Chain State Machine](#chain-state-machine) + - [Streaming Architecture (per page)](#streaming-architecture-per-page) + - [What the Receiver Does vs What It Doesn't](#what-the-receiver-does-vs-what-it-doesnt) + - [Pipeline Processors](#pipeline-processors) +- [Getting Started](#getting-started) + - [Scenario 1: Elasticsearch + Kibana Dashboard](#scenario-1-elasticsearch--kibana-dashboard) + - [Scenario 2: File Export (Testing / Archival)](#scenario-2-file-export-testing--archival) + - [Scenario 3: Console Debug (Development)](#scenario-3-console-debug-development) + - [Scenario 4: High-Throughput Production](#scenario-4-high-throughput-production) +- [Using Without Fleet (Headless Mode)](#using-without-fleet-headless-mode) +- [Live Test Results](#live-test-results) +- [Configuration Reference](#configuration-reference) +- [Telemetry](#telemetry) +- [Performance](#performance) + +## Architecture + +### Overview + +![Akamai SIEM Receiver Architecture — Overview](./img/architecture_overview.svg) + +![Receiver bodymap path](./img/architecture_raw.svg) + +The receiver places each raw Akamai JSON event into `LogRecord.Body` as a map with key `message`. It also writes `data_stream.{type,dataset,namespace}` to: + +- **Resource attributes** — used by the Elasticsearch exporter's dynamic routing to send the document to the correct data stream (e.g., `logs-akamai.siem-default`). Dynamic routing reads `data_stream.*` from resource attributes and is the default in current ES exporter versions. +- **The body map** — bodymap mode serializes only body content into the document, so these keys must also live in the body for Kibana filters like `data_stream.dataset:akamai.siem` to match. + +On the scope, the receiver sets `elastic.mapping.mode: bodymap`, which tells the Elasticsearch exporter to use bodymap serialization (write body map fields directly as document fields). The attribute lives on the `plog.Logs` data structure and travels end-to-end through the pipeline. + +`data_stream.*` defaults to `logs / akamai.siem / default`. Override per-environment via the `data_stream` config block on the receiver — see [Configuration Reference](#configuration-reference). + +#### What the ingest pipeline does + +Once the document lands in Elasticsearch, the Akamai integration's ingest pipeline takes over: + +1. `json` processor parses the `message` field (raw Akamai JSON) into structured fields. +2. Renames `message` → `event.original` (preserves the original). +3. Maps fields to ECS: `httpMessage.*` → `http.*`, `attackData.*` → custom ECS fields, `geo.*` → `source.geo.*`. +4. GeoIP enrichment on `source.ip`. +5. Base64 + URL decode on attack rule data. +6. `fingerprint` processor sets `_id` from event fields to deduplicate replays. + +Pre-built Kibana dashboards work immediately after the pipeline runs. + +### Data Flow + +1. **Poller** executes the three-branch chain state machine (offset drain, chain replay, new chain). +2. **EdgeGrid Auth** signs each HTTP request with HMAC-SHA256 per Akamai's specification. +3. **NDJSON Streaming** parses the response body through a bounded channel using a 1-line delay pattern: + - Scanner goroutine reads lines, sends to bounded channel (`stream_buffer_size`). + - Consumer batches events from the channel, calls `ConsumeLogs` per batch (`batch_size`). + - Back-pressure: when `ConsumeLogs` is slow, the channel fills, the scanner blocks. + - Peak memory bounded to `stream_buffer_size + batch_size` events regardless of page size. +4. **Cursor Store** persists chain state only after ALL batches in a page succeed. +5. **emitEvents** builds the `plog.Logs`: + - `LogRecord.Body` is a map: `{message: rawJSON, data_stream.type, data_stream.dataset, data_stream.namespace}`. + - Resource attributes carry `data_stream.{type,dataset,namespace}` for routing. + - Scope attributes carry `elastic.mapping.mode: bodymap` for ES exporter serialization. +6. The Elasticsearch exporter reads `data_stream.*` from resource attrs and routes to `logs--` (dynamic routing is the default in current versions). Bodymap serialization writes the body map fields directly into the document. + +### Chain State Machine + +The receiver implements a three-branch polling state machine to handle Akamai's offset-based and time-based chain pagination: + +``` + +------------------+ + | Start poll cycle | + +--------+---------+ + | + +--------------+--------------+ + | | | + [not caught up [not caught up [caught up + + valid offset] + no offset] or first run] + | | | + v v v + Branch 1: Branch 2: Branch 3: + DRAIN CHAIN REPLAY CHAIN NEW CHAIN + (offset-based) (time-based (time-based + with overlap) fresh window) + | | | + +--------------+--------------+ + | + +-------v--------+ + | Fetch page | + | Parse NDJSON | + | Emit events | + | Update cursor | + +-------+--------+ + | + +----------+---------+ + | | + [events < limit] [events == limit] + | | + v v + Chain drained Continue to + (caught up) next page +``` + +#### Error Recovery + +| Error | Recovery Action | +|---|---| +| 416 (offset expired) | Clear offset, replay chain with time window | +| 400 "invalid timestamp" | Retry up to `invalid_timestamp_retries`, then clear offset | +| 400 "from too old" | Clamp `from` to max lookback (12h), replay | +| 400 (other) | Non-recoverable, end poll cycle | +| Max recovery attempts reached | End poll cycle, log error | + +### Streaming Architecture (per page) + +``` +API Response Body + | + v +[Scanner Goroutine] + | 1-line delay: holds last line to check for offset context + v +bounded channel (stream_buffer_size=4) + | back-pressure: blocks scanner when full + v +[Batch Consumer] (on poller goroutine) + | accumulates batch_size=1000 events + v +emitEvents() → ConsumeLogs() + | repeated until channel drained + v +cursor persisted (only after all batches succeed) +``` + +### What the Receiver Does vs What It Doesn't + +| Responsibility | Owner | +|---|---| +| Poll Akamai SIEM API | Receiver | +| EdgeGrid authentication | Receiver | +| NDJSON parsing + offset tracking | Receiver | +| Chain state machine + cursor persistence | Receiver | +| Telemetry (16 metrics) | Receiver | +| Bodymap scope attribute + `data_stream.*` injection | Receiver | +| JSON field extraction | ES ingest pipeline | +| ECS mapping + GeoIP enrichment + base64/URL decode | ES ingest pipeline | +| Dashboards | Akamai integration (Kibana) | +| Data stream routing (resource attrs → index) | ES exporter (dynamic routing) | + +### Pipeline Processors + +The example configs in this README use standard OTel Collector processors. These are not part of the receiver — they are general-purpose pipeline components. + +- **`batch` processor** (optional, recommended) — Buffers log records and flushes them to the exporter in batches rather than one at a time. `timeout: 10s` flushes every 10 seconds or when the configured size limit is reached. Reduces bulk requests to Elasticsearch and improves throughput. The receiver writes the bodymap scope attribute and `data_stream.*` directly onto the `plog.Logs`, so they pass through the batch processor without extra configuration. + +You do not need a `transform` or `resource` processor for the common case: the receiver writes everything the Elasticsearch exporter needs. + +## Getting Started + +### Scenario 1: Elasticsearch + Kibana Dashboard + +The default and recommended path for Elastic users. Raw JSON flows to Elasticsearch where the Akamai integration's ingest pipeline handles all enrichment. Works with the existing Kibana Akamai SIEM dashboard. + +**Prerequisites:** +- Akamai SIEM API credentials (client token, client secret, access token). +- Akamai integration installed in Kibana (creates the ingest pipeline + index template). + +The minimum required components are the receiver and an exporter. Everything else in this example — `batch` processor, `file_storage` extension, `sending_queue.storage`, and the Prometheus metrics endpoint — is optional and called out inline. + +```yaml +receivers: + akamai_siem: + endpoint: "https://akab-xxxxx.luna.akamaiapis.net" + config_ids: "12345,67890" + authentication: + client_token: "${AKAMAI_CLIENT_TOKEN}" + client_secret: "${AKAMAI_CLIENT_SECRET}" + access_token: "${AKAMAI_ACCESS_TOKEN}" + storage: file_storage # optional — references the extension below for cursor persistence across restarts. Without it, the receiver re-fetches from `initial_lookback` on every start. + # data_stream defaults to logs / akamai.siem / default — override here if needed. + +processors: + # OPTIONAL: the batch processor amortizes flushes to the exporter. Without it, + # each `ConsumeLogs` call from the receiver becomes one Elasticsearch bulk + # request — workable but less efficient. + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + elasticsearch: + endpoints: + - "https://elasticsearch:9200" + api_key: "${ES_API_KEY}" + sending_queue: + enabled: true + storage: file_storage # optional — persists in-flight requests across restarts (at-least-once delivery). Drop this line to use an in-memory queue. + +extensions: + # OPTIONAL: required only if you reference `file_storage` above (cursor persistence + # and/or persistent sending queue). Drop this whole block if neither is needed. + file_storage: + directory: /var/lib/otelcol/storage + +service: + extensions: [file_storage] # only needed if the file_storage block is configured above + # OPTIONAL: exposes the Collector's own metrics on a Prometheus scrape endpoint. + # Drop this whole `telemetry` block if you don't need it; the receiver works without. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: "localhost" + port: 8888 + pipelines: + logs: + receivers: [akamai_siem] + processors: [batch] # drop `batch` from the list if you removed the processor above + exporters: [elasticsearch] +``` + +To split data per tenant or environment, override the receiver's `data_stream.dataset`: + +```yaml +receivers: + akamai_siem: + # ... + data_stream: + dataset: akamai.siem.tenant_a +``` + +### Scenario 2: File Export (Testing / Archival) + +Write events to NDJSON files for offline analysis or archival. The body map (including `data_stream.*` keys) is preserved in the file output. + +```yaml +receivers: + akamai_siem: + endpoint: "https://akab-xxxxx.luna.akamaiapis.net" + config_ids: "12345" + authentication: + client_token: "${AKAMAI_CLIENT_TOKEN}" + client_secret: "${AKAMAI_CLIENT_SECRET}" + access_token: "${AKAMAI_ACCESS_TOKEN}" + poll_interval: 5m + initial_lookback: 1h + +exporters: + file: + path: "/var/log/akamai-siem-events.jsonl" + # Production-safe defaults: rotate to keep the disk bounded and compress + # rotated files. Without rotation, a 24/7 ingestion run will eventually + # fill the disk. + rotation: + max_megabytes: 100 # roll over once the active file hits 100 MB + max_backups: 10 # keep at most 10 historical files + max_days: 7 # delete anything older than a week + compression: zstd # zstd-compress rotated files (small, fast) + flush_interval: 1s # bounded latency for the active file + +service: + pipelines: + logs: + receivers: [akamai_siem] + exporters: [file] +``` + +### Scenario 3: Console Debug (Development) + +Print events to stdout for development and troubleshooting. No infrastructure needed. + +```yaml +receivers: + akamai_siem: + endpoint: "https://akab-xxxxx.luna.akamaiapis.net" + config_ids: "12345" + authentication: + client_token: "${AKAMAI_CLIENT_TOKEN}" + client_secret: "${AKAMAI_CLIENT_SECRET}" + access_token: "${AKAMAI_ACCESS_TOKEN}" + poll_interval: 30s + initial_lookback: 1h + event_limit: 100 + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + logs: + receivers: [akamai_siem] + exporters: [debug] +``` + +### Scenario 4: High-Throughput Production + +Tuned for maximum throughput with large batch sizes and persistent queuing. As with Scenario 1, only the receiver and exporter are required — `batch`, `file_storage`, `sending_queue`, `retry`, and the metrics endpoint are all optional. They're recommended for a production-grade deployment and called out inline. + +```yaml +receivers: + akamai_siem: + endpoint: "https://akab-xxxxx.luna.akamaiapis.net" + config_ids: "12345,67890,11111" + authentication: + client_token: "${AKAMAI_CLIENT_TOKEN}" + client_secret: "${AKAMAI_CLIENT_SECRET}" + access_token: "${AKAMAI_ACCESS_TOKEN}" + poll_interval: 10s + event_limit: 100000 + batch_size: 5000 + stream_buffer_size: 16 + offset_ttl: 60s + storage: file_storage # optional — references the extension below + timeout: 120s + +processors: + # OPTIONAL: amortizes flushes to the exporter. Recommended at high throughput + # so the exporter sees few large bulk requests rather than many small ones. + batch: + timeout: 5s + send_batch_size: 5000 + send_batch_max_size: 10000 + +exporters: + elasticsearch: + endpoints: + - "https://es-node1:9200" + - "https://es-node2:9200" + - "https://es-node3:9200" + api_key: "${ES_API_KEY}" + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 10000 + storage: file_storage # optional — persists in-flight requests across restarts (at-least-once delivery) + retry: + enabled: true + max_elapsed_time: 300s + +extensions: + # OPTIONAL: required only if you reference `file_storage` above (cursor + # persistence and/or persistent sending queue). + file_storage: + directory: /var/lib/otelcol/storage + # In a 24/7 producer the persistent queue grows in bursts (back-pressure + # from Elasticsearch, network blips). On-rebound compaction reclaims disk + # after spikes; without it the BoltDB file only grows. + compaction: + on_rebound: true + directory: /var/lib/otelcol/storage + rebound_needed_threshold_mib: 100 + rebound_trigger_threshold_mib: 10 + fsync: true # durability over throughput — survive a hard crash mid-write + +service: + extensions: [file_storage] # only needed if the file_storage block is configured above + # OPTIONAL: exposes the Collector's own metrics on a Prometheus scrape endpoint. + # Drop this whole `telemetry` block if you don't need it; the receiver works without. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: "localhost" + port: 8888 + pipelines: + logs: + receivers: [akamai_siem] + processors: [batch] # drop `batch` from the list if you removed the processor above + exporters: [elasticsearch] +``` + +### Pipeline Component Reference + +The example pipelines compose standard Collector components. Each one is independently documented upstream; this section explains the knobs the examples set and why. **Only the receiver and an exporter are required** — `batch`, `file_storage`, the metrics endpoint, and the exporter `sending_queue`/`retry` blocks are all optional. + +**`batch` processor** (optional) — `processors.batch` + +- `timeout` (default `200ms` upstream; `5–10s` in the examples) — flush even if the batch is below `send_batch_size`. Higher values amortize flush overhead at the cost of latency. +- `send_batch_size` (default `8192`) — target batch size. The processor flushes when the accumulated record count reaches this. Set this near the receiver's `batch_size` so the processor doesn't have to re-batch. +- `send_batch_max_size` (no default; only used when set) — hard cap. If a single `ConsumeLogs` call upstream contains more records than this, the processor splits it. Useful when the upstream Elasticsearch `bulk` API or proxy has a per-request size limit. + +**`elasticsearch` exporter** — `exporters.elasticsearch` + +- `endpoints` — list of ES URLs; the exporter load-balances across them. Use multiple entries for HA — a single failed node is then transparently skipped. +- `api_key` vs `user`+`password` — pick one. API keys are preferred in production (per-key permissions, no plaintext password). The value is base64-encoded `id:api_key`. +- `tls.insecure_skip_verify` — only for local testing with self-signed ES. Never set this against a real cluster. +- `sending_queue.enabled` — buffers in-flight bulk requests. Defaults to `true` upstream; setting it explicitly for clarity is fine. +- `sending_queue.storage` — references a storage extension (e.g. `file_storage`) so the queue persists across collector restarts. Without this, an in-memory queue is used and queued events are lost on restart. +- `sending_queue.queue_size` — max bulk requests the queue will hold. The default (`1000`) is fine for steady-state ingestion; raise it for bursty traffic where a brief ES outage shouldn't drop data. +- `sending_queue.num_consumers` — concurrent senders draining the queue. The default (`10`) saturates most ES clusters; raise only if you've measured the queue draining slower than it fills. +- `retry.enabled` + `retry.max_elapsed_time` — exponential-backoff retry for transient failures (5xx, 429). `max_elapsed_time` caps the total retry budget per request. + +**`file_storage` extension** (optional) — `extensions.file_storage` + +- `directory` — where the BoltDB-backed key-value file lives. Used by both the receiver's cursor and the exporter's persistent sending queue. Both live in the same DB but in separate keyspaces — no conflict. +- `compaction.on_rebound` — when enabled, BoltDB is compacted after a spike (queue drained from large to small). Without this, the file only grows. Strongly recommended for long-running deployments. +- `fsync` — fsync each write. Trades throughput for durability. Set to `true` if a power loss must not lose the last few queued events; leave default for higher-throughput-but-eventually-durable setups. + +**`debug` exporter** — `exporters.debug` + +- `verbosity` — one of `basic` (default), `normal`, `detailed`. Use `detailed` to see full record contents in stdout; `basic` only logs aggregate counts. + +**`file` exporter** — `exporters.file` + +- `path` — output file. Without `rotation`, this file grows unbounded. +- `rotation.max_megabytes` / `max_backups` / `max_days` — bounded retention. Without these, a 24/7 producer will fill the disk. +- `compression` — `zstd` is fast and small; rotated files compress well (NDJSON is repetitive). +- `flush_interval` — how often the active file is flushed to disk. Lower values reduce window of loss on crash. + +## Using Without Fleet (Headless Mode) + +You can use this receiver without Kibana Fleet or an agent policy. This is the typical setup when running the OTel Collector directly — for example, in a containerized environment, on a VM, or locally for testing. + +The only requirement is that the Akamai integration assets (index template + ingest pipeline) are installed in Elasticsearch. There are two ways to do this. + +### Option 1: Install assets via Fleet API (no agent policy) + +Fleet can install integration assets without attaching them to a policy. Send a single POST to the Fleet package API: + +```bash +curl -s -u elastic:changeme \ + -X POST "https://localhost:5601/api/fleet/epm/packages/akamai/2.33.2" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json" \ + -d '{}' | jq .result +``` + +This installs the index template `logs-akamai.siem-*` and the ingest pipeline without creating any agent policy. + +Check the available version first: + +```bash +curl -s -u elastic:changeme \ + "https://localhost:5601/api/fleet/epm/packages/akamai" | jq .item.version +``` + +### Option 2: Install from Kibana UI + +Go to **Kibana → Integrations → Akamai → Add Akamai SIEM** and complete the wizard. You can use a dummy agent policy — the integration assets are installed regardless. + +### Production: Elasticsearch via HTTPS + API Key + +```yaml +exporters: + elasticsearch: + endpoints: + - "https://your-cluster.es.io:9200" + api_key: "${ES_API_KEY}" # base64-encoded id:api_key + sending_queue: + enabled: true + storage: file_storage + retry: + enabled: true + max_elapsed_time: 300s +``` + +To create an API key scoped to this index: + +```bash +curl -s -u elastic:changeme \ + -X POST "https://localhost:9200/_security/api_key" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "akamai-siem-receiver", + "role_descriptors": { + "akamai_writer": { + "cluster": ["monitor"], + "index": [{ + "names": ["logs-akamai.*"], + "privileges": ["create_index", "create", "auto_configure"] + }] + } + } + }' | jq -r '"\(.id):\(.api_key)" | @base64' +``` + +### Local Testing: Docker Elasticsearch + +Run Elasticsearch locally to test the receiver end-to-end: + +```bash +docker run -d --name es-local \ + -e "discovery.type=single-node" \ + -e "ELASTIC_PASSWORD=changeme" \ + -e "xpack.security.enabled=true" \ + -p 9200:9200 \ + docker.elastic.co/elasticsearch/elasticsearch:8.17.0 + +until curl -s -u elastic:changeme https://localhost:9200/_cluster/health \ + --cacert /tmp/es-certs/ca.crt | jq -e '.status != "red"' > /dev/null; do + sleep 2 +done +``` + +Minimal test config (`akamai-test.yaml`): + +```yaml +receivers: + akamai_siem: + endpoint: "https://akab-xxxxx.luna.akamaiapis.net" + config_ids: "12345" + authentication: + client_token: "${AKAMAI_CLIENT_TOKEN}" + client_secret: "${AKAMAI_CLIENT_SECRET}" + access_token: "${AKAMAI_ACCESS_TOKEN}" + poll_interval: 1m + initial_lookback: 1h + event_limit: 1000 + +exporters: + elasticsearch: + endpoints: ["https://localhost:9200"] + user: "elastic" + password: "changeme" + tls: + insecure_skip_verify: true + debug: + verbosity: detailed + +service: + telemetry: + logs: + level: debug + metrics: + level: none + pipelines: + logs: + receivers: [akamai_siem] + exporters: [elasticsearch, debug] +``` + +Run the collector: + +```bash +./otelcol --config akamai-test.yaml +``` + +Verify events are arriving: + +```bash +curl -s -u elastic:changeme \ + "https://localhost:9200/logs-akamai.siem-default/_count" \ + --insecure | jq .count +``` + +### Checking Index and Pipeline Health + +```bash +# Check index template was installed +curl -s -u elastic:changeme \ + "https://localhost:9200/_index_template/logs-akamai.siem" --insecure | jq . + +# Check ingest pipeline +curl -s -u elastic:changeme \ + "https://localhost:9200/_ingest/pipeline/logs-akamai.siem-*" --insecure | jq keys + +# Check data stream stats +curl -s -u elastic:changeme \ + "https://localhost:9200/_data_stream/logs-akamai.siem-default/_stats" --insecure | jq . +``` + +## Live Test Results + +The following document was captured from a live run against the Akamai SIT environment with a local Elasticsearch 9.3.1 stack. + +Data stream: `logs-akamai.siem-default`. The receiver writes the bodymap scope attribute and `data_stream.*` directly; the user pipeline is `[batch]`. The Akamai ingest pipeline parses the raw JSON in `message` into full ECS fields: + +```json +{ + "_index": ".ds-logs-akamai.siem-default-2026.03.29-000001", + "_id": "1774777751-zW7QLk7MmGNbuha8XEVw2iitcJc=", + "_source": { + "@timestamp": "2026-03-29T09:49:11.000Z", + "ecs": { "version": "8.11.0" }, + "event": { + "category": ["network"], + "id": "4f9ec66f85e9a0a8", + "kind": "event", + "start": "2026-03-29T09:49:11.000Z" + }, + "http": { + "request": { "id": "4f9ec66f85e9a0a8", "method": "DELETE" }, + "response": { "bytes": 39878, "status_code": 223 }, + "version": "2.0" + }, + "network": { "protocol": "http", "transport": "tcp" }, + "observer": { "type": "proxy", "vendor": "akamai" }, + "source": { + "address": "207.220.32.43", + "as": { "number": 16148 }, + "geo": { "city_name": "San Francisco", "country_iso_code": "US", "region_iso_code": "-CA" }, + "ip": "207.220.32.43" + }, + "client": { + "address": "207.220.32.43", + "ip": "207.220.32.43", + "geo": { "city_name": "San Francisco", "country_iso_code": "US" } + }, + "url": { + "domain": "example66.com", + "full": "example66.com/api/v5/resource?id=2752", + "path": "/api/v5/resource", + "port": 80, + "query": "id=2752" + }, + "akamai": { + "siem": { + "config_id": "1", + "policy_id": "policy_8", + "rule_actions": ["deny", "alert"], + "rule_tags": ["owasp", "sql"], + "rules": [ + { "rules": "950089", "ruleMessages": "Command Injection attempt detected", "ruleActions": "deny" }, + { "rules": "950062", "ruleMessages": "SQL Injection attempt detected", "ruleActions": "deny" } + ] + } + } + } +} +``` + +The `_id` (`1774777751-zW7QLk7MmGNbuha8XEVw2iitcJc=`) is set by the ingest pipeline's fingerprint processor for event deduplication. The Kibana Akamai SIEM dashboard works immediately. + +## Configuration Reference + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `endpoint` | string | (required) | Akamai API host URL | +| `config_ids` | string | (required) | Comma or semicolon-separated security configuration IDs | +| `authentication.client_token` | string | (required) | EdgeGrid client token | +| `authentication.client_secret` | string | (required) | EdgeGrid client secret | +| `authentication.access_token` | string | (required) | EdgeGrid access token | +| `data_stream.type` | string | `logs` | Value written to `data_stream.type` on resource attrs and body map | +| `data_stream.dataset` | string | `akamai.siem` | Value written to `data_stream.dataset`. Override per tenant/environment. | +| `data_stream.namespace` | string | `default` | Value written to `data_stream.namespace` | +| `poll_interval` | duration | `1m` | Time between polling cycles | +| `initial_lookback` | duration | `12h` | Lookback window for first poll (max 12h, Akamai limit) | +| `event_limit` | int | `10000` | Max events per API request (max 600000) | +| `offset_ttl` | duration | `120s` | Max age of stored offset before proactive replay (0 disables) | +| `max_recovery_attempts` | int | `3` | Max consecutive recovery actions per poll cycle (0 disables) | +| `invalid_timestamp_retries` | int | `2` | Immediate retries for 400 "invalid timestamp" errors | +| `batch_size` | int | `1000` | Events per ConsumeLogs call. Controls memory per batch. | +| `stream_buffer_size` | int | `4` | Bounded channel capacity between NDJSON scanner and batch consumer. Controls back-pressure. | +| `timeout` | duration | `60s` | HTTP request timeout. Plus other `confighttp.ClientConfig` fields (`tls`, `headers`, `compression`, `proxy_url`, …) at the receiver root. | +| `storage` | component.ID | (nil) | Storage extension for cursor persistence (e.g., `file_storage`). Optional — when unset the receiver still tracks chain state in memory and runs without errors, but every collector restart re-fetches from `initial_lookback` since the cursor doesn't survive the process. | + +> **Note on `http.auth`**: configuring an `auth` authenticator on the HTTP client is rejected at validation time. The receiver wraps the configured transport with the EdgeGrid signer, so a chained authenticator would run after EdgeGrid signs the request and silently invalidate the signature. Use the top-level `authentication` block (`client_token`, `client_secret`, `access_token`) for EdgeGrid HMAC-SHA256 credentials — that is the only supported auth mechanism for this receiver. + +The receiver also embeds [`confighttp.ClientConfig`](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/README.md) for TLS, proxy, and advanced HTTP settings. + +### Tuning Guide + +These knobs operate at different layers and are easy to confuse. The defaults work for most setups; this section is for when you need to depart from them. + +**`event_limit` vs `batch_size`** — two different things. + +- `event_limit` is the **`limit` query parameter** sent to Akamai per HTTP request. It caps the number of events in a single API response page. Higher values mean fewer HTTP requests for a given event volume but larger responses (more memory while streaming, longer time-to-first-byte). The Akamai-imposed maximum is 600,000. +- `batch_size` is how many events the receiver groups into a single `ConsumeLogs` call to the downstream pipeline. A page of 100,000 events with `batch_size: 1000` results in 100 separate `ConsumeLogs` calls. Higher values reduce per-batch overhead but bound peak memory: only `batch_size` events sit in `plog.Logs` form at once. + +Rule of thumb: pick `event_limit` based on how often you poll (see below). Pick `batch_size` based on how big you want each downstream flush to be — match it to your exporter's `send_batch_size` to avoid the batch processor having to re-batch. + +**`stream_buffer_size`** — how many parsed events can sit between the NDJSON scanner goroutine and the batch consumer before the scanner blocks. This is the receiver's primary back-pressure mechanism: when the downstream `ConsumeLogs` is slow (Elasticsearch indexer at capacity, sending queue full), the scanner stops reading from the API response. This bounds peak memory regardless of page size — a 600k-event page still only holds `stream_buffer_size + batch_size` events in memory. The default of `4` is intentionally tiny; raising it does not increase throughput (the receiver is I/O-bound on the upstream gzip stream) but does increase peak memory under back-pressure. Leave at the default unless you're benchmarking. + +**`poll_interval` vs `initial_lookback`** — when each one matters. + +- `poll_interval` is the **steady-state** cadence: after each poll cycle drains, the receiver sleeps this long before starting the next. Shorter intervals mean fresher data but more API calls. Akamai's API is offset-based, so a busy poll cycle can take longer than `poll_interval` itself — that's fine; the next tick fires immediately when the current cycle finishes. +- `initial_lookback` only matters on the **first run with no persisted cursor** (or if `storage:` is unset). It's the time window the first poll fetches. After the first cycle, the receiver tracks chain progression via the cursor, so this value is ignored. Capped at 12h by the Akamai API. + +If you set `storage: file_storage` and lose the cursor (file deleted, fresh container), the next start re-fetches `initial_lookback` of history — which is by design, but be aware that long lookbacks on restart can produce a large initial spike. + +**`offset_ttl`** — how stale a stored offset can be before the receiver gives up on it and replays the chain window. + +Akamai offsets are valid for a limited window. If the collector pauses long enough (network blip, host suspended, GC pause) that the next poll's offset is past Akamai's retention, the API returns a 416 and the receiver has to recover by replaying with a time window. Setting `offset_ttl` proactively triggers that replay before the API rejects the offset, reducing recovery latency. Trade-off: too low and you replay unnecessarily (re-fetching events you already saw); too high and you risk hitting 416 first. The default `120s` is conservative. + +Setting `offset_ttl: 0` disables proactive replay entirely — the receiver only replays in response to actual 416 responses. + +**`max_recovery_attempts`** — the safety valve. + +Caps how many consecutive recovery actions (offset replay, timestamp retry, from-clamp) the receiver will take in a single poll cycle before giving up and waiting for the next `poll_interval`. Prevents runaway loops if Akamai is in a degraded state. Default `3` is a sane balance; setting `0` disables the cap (not recommended in production). + +**`data_stream.dataset` override** — when to set it. + +Useful for splitting traffic across data streams: per Akamai customer/tenant (`akamai.siem.tenant_a` / `akamai.siem.tenant_b`), per environment (`akamai.siem.staging` vs `akamai.siem.prod`), or to sidestep an existing Akamai-integration data stream during a migration. Each unique value produces a distinct data stream — `logs--`. The Elastic Akamai integration only owns `akamai.siem` and `akamai.siem.*`, so prefer dot-suffixed variants over a fully different name to keep the integration's index template and ingest pipeline matching. + +## Telemetry + +The receiver exposes the following OTel metrics via the collector's telemetry endpoint: + +| Metric | Type | Description | +|---|---|---| +| `otelcol_akamai_siem_requests` | counter | Total API requests made | +| `otelcol_akamai_siem_request_errors` | counter | Total failed API requests | +| `otelcol_akamai_siem_events_received` | counter | Total events received from API | +| `otelcol_akamai_siem_events_emitted` | counter | Total events successfully forwarded to the downstream consumer via `ConsumeLogs` | +| `otelcol_akamai_siem_offset_expired` | counter | Total 416 offset out of range errors | +| `otelcol_akamai_siem_offset_ttl_drops` | counter | Total proactive offset TTL expirations | +| `otelcol_akamai_siem_recovery_attempts` | counter | Total recovery actions taken | +| `otelcol_akamai_siem_invalid_timestamp_retries` | counter | Total HMAC timestamp retries | +| `otelcol_akamai_siem_pages_processed` | counter | Total API response pages processed | +| `otelcol_akamai_siem_cursor_persists` | counter | Total successful cursor persist operations | +| `otelcol_akamai_siem_bytes_received` | counter | Total bytes received from the Akamai SIEM API | +| `otelcol_akamai_siem_request_duration` | histogram | API roundtrip latency (seconds) | +| `otelcol_akamai_siem_poll_duration` | histogram | Full poll iteration duration (seconds) | +| `otelcol_akamai_siem_events_per_second` | histogram | Events per second throughput per poll cycle | +| `otelcol_akamai_siem_page_processing_time` | histogram | Per-page processing time: NDJSON parse + body-map construction + ConsumeLogs (seconds) | +| `otelcol_akamai_siem_events_per_page` | histogram | Events received per API response page | + +These metrics are emitted only if the Collector's telemetry endpoint is configured. The endpoint itself is **optional** — the receiver works fine without it; you just won't be able to scrape its metrics. To enable, add this block to `service`: + +```yaml +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: "localhost" + port: 8888 +``` + +> **Note**: bind the metrics endpoint to `localhost` unless you intend to expose it on all interfaces. + +## Performance + +Benchmarked against a live Akamai SIEM API endpoint with 100,000 events per page, nop exporter (measures receiver-only throughput). Apple M1 Max, 32 GB RAM, Go 1.24.7. + +| Metric | Value | +|---|---| +| **EPS** | ~15,600 | +| CPU/event | 0.009ms | +| RSS | 105 MB | +| Alloc/event | 1.3 KB | +| CPU utilization | 12.8% | +| Goroutines | 12 | + +EPS is I/O-bound — gzip decompression and NDJSON streaming dominate. Body-map construction is effectively zero CPU per event. + +### Key Observations + +- **Batch size (500-5000) does not affect EPS.** RSS increases only at batch_size=5000 (+10 MB). +- **Buffer size (1-8) does not affect EPS or RSS.** Default of 4 is optimal. +- **Peak memory is bounded by `batch_size`, not page size.** A 600k-event page is processed as 600 batches of 1,000 — memory stays constant. + +### Running Benchmarks + +```bash +cd receiver/akamaisiemreceiver +go test -bench=BenchmarkEmitEvents -benchtime=1s -run='^$' . +``` diff --git a/receiver/akamaisiemreceiver/benchmark_test.go b/receiver/akamaisiemreceiver/benchmark_test.go new file mode 100644 index 000000000..519459ad0 --- /dev/null +++ b/receiver/akamaisiemreceiver/benchmark_test.go @@ -0,0 +1,182 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +// --- EmitEvents benchmarks: measure plog.Logs construction cost --- + +func BenchmarkEmitEvents_1(b *testing.B) { benchEmit(b, 1) } +func BenchmarkEmitEvents_1000(b *testing.B) { benchEmit(b, 1000) } +func BenchmarkEmitEvents_100000(b *testing.B) { benchEmit(b, 100000) } +func BenchmarkEmitEvents_600000(b *testing.B) { benchEmit(b, 600000) } + +func benchEmit(b *testing.B, n int) { + b.Helper() + events := loadBenchEvents(b, n) + rcv := benchReceiver(b) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := rcv.emitEvents(context.Background(), events); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(n), "events/op") +} + +// --- FullPoll benchmarks: HTTP fetch + NDJSON parse + emit --- + +func BenchmarkFullPoll_100(b *testing.B) { benchFullPoll(b, 100) } +func BenchmarkFullPoll_10000(b *testing.B) { benchFullPoll(b, 10000) } +func BenchmarkFullPoll_100000(b *testing.B) { benchFullPoll(b, 100000) } +func BenchmarkFullPoll_600000(b *testing.B) { benchFullPoll(b, 600000) } + +func benchFullPoll(b *testing.B, eventCount int) { + b.Helper() + ndjsonBody := buildNDJSON(b, eventCount) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(ndjsonBody) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "bench" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour // don't auto-poll + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + if err != nil { + b.Fatal(err) + } + if err := rcv.Start(context.Background(), componenttest.NewNopHost()); err != nil { + b.Fatal(err) + } + + // Wait for first poll to complete. + deadline := time.Now().Add(10 * time.Second) + for sink.LogRecordCount() == 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + + // Reset sink and timer for benchmark iterations. + sink.Reset() + + // We can't re-trigger polls easily, so benchmark the emitEvents path directly. + akRcv := rcv.(*akamaiReceiver) + events := loadBenchEvents(b, eventCount) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + sink.Reset() + if err := akRcv.emitEvents(context.Background(), events); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(eventCount), "events/op") + + _ = rcv.Shutdown(context.Background()) +} + +// --- helpers --- + +// loadBenchEvents creates n copies of a realistic Akamai event from testdata. +func loadBenchEvents(b testing.TB, n int) []string { + b.Helper() + data, err := os.ReadFile("testdata/siem_response.ndjson") + if err != nil { + b.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + // Use only event lines (not offset context). + var eventLines []string + for _, line := range lines { + if line == "" || strings.Contains(line, `"offset"`) { + continue + } + eventLines = append(eventLines, line) + } + if len(eventLines) == 0 { + b.Fatal("no events in testdata") + } + + events := make([]string, n) + for i := 0; i < n; i++ { + events[i] = eventLines[i%len(eventLines)] + } + return events +} + +// buildNDJSON creates NDJSON response body with n events + offset context. +func buildNDJSON(b testing.TB, n int) []byte { + b.Helper() + events := loadBenchEvents(b, n) + var sb strings.Builder + for _, e := range events { + sb.WriteString(e) + sb.WriteString("\n") + } + sb.WriteString(fmt.Sprintf(`{"offset":"bench-cursor","total":%d,"limit":%d}`, n, n+1)) + sb.WriteString("\n") + return []byte(sb.String()) +} + +// benchReceiver creates a receiver wired to a nop consumer for benchmarking. +func benchReceiver(b testing.TB) *akamaiReceiver { + b.Helper() + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = "https://bench.example.com" + cfg.ConfigIDs = "bench" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + rcv, err := newAkamaiReceiver(cfg, set, sink) + if err != nil { + b.Fatal(err) + } + return rcv +} diff --git a/receiver/akamaisiemreceiver/config.go b/receiver/akamaisiemreceiver/config.go new file mode 100644 index 000000000..0a8f75bb1 --- /dev/null +++ b/receiver/akamaisiemreceiver/config.go @@ -0,0 +1,198 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver" + +import ( + "errors" + "fmt" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" +) + +const ( + defaultPollInterval = time.Minute + defaultInitialLookback = 12 * time.Hour + maxInitialLookback = 12 * time.Hour + defaultEventLimit = 10000 + maxEventLimit = 600000 + defaultOffsetTTL = 120 * time.Second + defaultMaxRecovery = 3 + defaultInvalidTSRetry = 2 + defaultBatchSize = 1000 + defaultStreamBufferSize = 4 + + defaultDataStreamType = "logs" + defaultDataStreamDataset = "akamai.siem" + defaultDataStreamNamespace = "default" +) + +// Config defines the configuration for the Akamai SIEM receiver. +type Config struct { + // HTTP exposes the standard confighttp.ClientConfig fields (endpoint, tls, + // timeout, headers, compression, …) at the receiver root via the squash tag. + // Use `endpoint:` directly under the receiver to set the Akamai API host + // (e.g. "https://akab-xxx.luna.akamaiapis.net"). The `auth` field is rejected + // in Validate — credentials must be configured via the authentication block. + HTTP confighttp.ClientConfig `mapstructure:",squash"` + + // ConfigIDs is a semicolon or comma-separated list of security configuration IDs. + ConfigIDs string `mapstructure:"config_ids"` + + // Authentication holds Akamai EdgeGrid HMAC-SHA256 credentials. + Authentication EdgeGridAuth `mapstructure:"authentication"` + + // PollInterval is the time between polling cycles. + PollInterval time.Duration `mapstructure:"poll_interval"` + + // InitialLookback is the lookback window for the first poll. Max 12h (Akamai limit). + InitialLookback time.Duration `mapstructure:"initial_lookback"` + + // EventLimit is the max events per API request. Max 600000. + EventLimit int `mapstructure:"event_limit"` + + // OffsetTTL is the max age of a stored offset before proactive replay. Zero disables. + OffsetTTL time.Duration `mapstructure:"offset_ttl"` + + // MaxRecoveryAttempts caps consecutive recovery actions per poll cycle. Zero disables. + MaxRecoveryAttempts int `mapstructure:"max_recovery_attempts"` + + // InvalidTimestampRetries is the number of immediate retries for 400 "invalid timestamp". + InvalidTimestampRetries int `mapstructure:"invalid_timestamp_retries"` + + // BatchSize is the number of events per ConsumeLogs call. Default 1000. + BatchSize int `mapstructure:"batch_size"` + + // StreamBufferSize is the bounded channel capacity between the NDJSON scanner + // and the batch consumer. Controls back-pressure. Default 4. + StreamBufferSize int `mapstructure:"stream_buffer_size"` + + // DataStream controls the data_stream.{type,dataset,namespace} fields the + // receiver writes to resource attributes (for ES exporter routing) and to + // the body map (for Kibana filters in bodymap mode). + DataStream DataStreamConfig `mapstructure:"data_stream"` + + // StorageID references a storage extension for persisting cursor state across + // restarts. If nil, cursor persistence is disabled and the receiver starts fresh. + // Use with the file_storage extension for file-based persistence. + StorageID *component.ID `mapstructure:"storage"` +} + +// DataStreamConfig controls the Elasticsearch data stream this receiver targets. +// Defaults to logs / akamai.siem / default. +type DataStreamConfig struct { + Type string `mapstructure:"type"` + Dataset string `mapstructure:"dataset"` + Namespace string `mapstructure:"namespace"` +} + +// EdgeGridAuth holds Akamai EdgeGrid HMAC-SHA256 credentials. +type EdgeGridAuth struct { + ClientToken configopaque.String `mapstructure:"client_token"` + ClientSecret configopaque.String `mapstructure:"client_secret"` + AccessToken configopaque.String `mapstructure:"access_token"` +} + +func createDefaultConfig() component.Config { + return &Config{ + PollInterval: defaultPollInterval, + InitialLookback: defaultInitialLookback, + EventLimit: defaultEventLimit, + OffsetTTL: defaultOffsetTTL, + MaxRecoveryAttempts: defaultMaxRecovery, + InvalidTimestampRetries: defaultInvalidTSRetry, + BatchSize: defaultBatchSize, + StreamBufferSize: defaultStreamBufferSize, + DataStream: DataStreamConfig{ + Type: defaultDataStreamType, + Dataset: defaultDataStreamDataset, + Namespace: defaultDataStreamNamespace, + }, + HTTP: confighttp.ClientConfig{ + Timeout: 60 * time.Second, + }, + } +} + +// Validate checks the configuration for correctness. +func (c *Config) Validate() error { + if c.HTTP.Endpoint == "" { + return errors.New("endpoint is required") + } + if c.ConfigIDs == "" { + return errors.New("config_ids is required") + } + if string(c.Authentication.ClientToken) == "" { + return errors.New("auth.client_token is required") + } + if string(c.Authentication.ClientSecret) == "" { + return errors.New("auth.client_secret is required") + } + if string(c.Authentication.AccessToken) == "" { + return errors.New("auth.access_token is required") + } + if c.PollInterval <= 0 { + return errors.New("poll_interval must be greater than 0") + } + if c.InitialLookback <= 0 { + return errors.New("initial_lookback must be greater than 0") + } + if c.InitialLookback > maxInitialLookback { + return fmt.Errorf("initial_lookback cannot exceed %v (Akamai API limit)", maxInitialLookback) + } + if c.EventLimit <= 0 { + return errors.New("event_limit must be greater than 0") + } + if c.EventLimit > maxEventLimit { + return fmt.Errorf("event_limit cannot exceed %d", maxEventLimit) + } + if c.OffsetTTL < 0 { + return errors.New("offset_ttl must be non-negative") + } + if c.MaxRecoveryAttempts < 0 { + return errors.New("max_recovery_attempts must be non-negative") + } + if c.InvalidTimestampRetries < 0 { + return errors.New("invalid_timestamp_retries must be non-negative") + } + if c.BatchSize <= 0 { + return errors.New("batch_size must be greater than 0") + } + if c.StreamBufferSize <= 0 { + return errors.New("stream_buffer_size must be greater than 0") + } + if c.DataStream.Type == "" { + return errors.New("data_stream.type is required") + } + if c.DataStream.Dataset == "" { + return errors.New("data_stream.dataset is required") + } + if c.DataStream.Namespace == "" { + return errors.New("data_stream.namespace is required") + } + // Reject http.auth: the receiver wraps the configured transport with the + // EdgeGrid signer, so any http.auth authenticator would run after EdgeGrid + // and silently invalidate the request signature. Use the top-level + // authentication block for EdgeGrid HMAC-SHA256 credentials instead. + if c.HTTP.Auth.HasValue() { + return errors.New("http.auth is not supported; this receiver authenticates via Akamai EdgeGrid HMAC-SHA256 (configure credentials under the authentication block)") + } + return nil +} diff --git a/receiver/akamaisiemreceiver/config_test.go b/receiver/akamaisiemreceiver/config_test.go new file mode 100644 index 000000000..ad9c81c2f --- /dev/null +++ b/receiver/akamaisiemreceiver/config_test.go @@ -0,0 +1,145 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configauth" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configoptional" +) + +func validConfig() *Config { + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = "https://akab-test.luna.akamaiapis.net" + cfg.ConfigIDs = "12345" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + return cfg +} + +func TestConfigValidate_Valid(t *testing.T) { + cfg := validConfig() + require.NoError(t, cfg.Validate()) +} + +func TestConfigValidate_MissingEndpoint(t *testing.T) { + cfg := validConfig() + cfg.HTTP.Endpoint = "" + assert.ErrorContains(t, cfg.Validate(), "endpoint is required") +} + +func TestConfigValidate_MissingConfigIDs(t *testing.T) { + cfg := validConfig() + cfg.ConfigIDs = "" + assert.ErrorContains(t, cfg.Validate(), "config_ids is required") +} + +func TestConfigValidate_MissingAuth(t *testing.T) { + cfg := validConfig() + cfg.Authentication.ClientToken = "" + assert.ErrorContains(t, cfg.Validate(), "auth.client_token is required") + + cfg = validConfig() + cfg.Authentication.ClientSecret = "" + assert.ErrorContains(t, cfg.Validate(), "auth.client_secret is required") + + cfg = validConfig() + cfg.Authentication.AccessToken = "" + assert.ErrorContains(t, cfg.Validate(), "auth.access_token is required") +} + +func TestConfigValidate_InvalidPollInterval(t *testing.T) { + cfg := validConfig() + cfg.PollInterval = 0 + assert.ErrorContains(t, cfg.Validate(), "poll_interval must be greater than 0") +} + +func TestConfigValidate_InitialLookbackExceeds12h(t *testing.T) { + cfg := validConfig() + cfg.InitialLookback = 13 * time.Hour + assert.ErrorContains(t, cfg.Validate(), "initial_lookback cannot exceed") +} + +func TestConfigValidate_EventLimitExceedsMax(t *testing.T) { + cfg := validConfig() + cfg.EventLimit = 700000 + assert.ErrorContains(t, cfg.Validate(), "event_limit cannot exceed") +} + +func TestConfigValidate_NegativeOffsetTTL(t *testing.T) { + cfg := validConfig() + cfg.OffsetTTL = -1 + assert.ErrorContains(t, cfg.Validate(), "offset_ttl must be non-negative") +} + +func TestConfigDefaults(t *testing.T) { + cfg := createDefaultConfig().(*Config) + assert.Equal(t, time.Minute, cfg.PollInterval) + assert.Equal(t, 12*time.Hour, cfg.InitialLookback) + assert.Equal(t, 10000, cfg.EventLimit) + assert.Equal(t, 120*time.Second, cfg.OffsetTTL) + assert.Equal(t, 3, cfg.MaxRecoveryAttempts) + assert.Equal(t, 2, cfg.InvalidTimestampRetries) + assert.Equal(t, 1000, cfg.BatchSize) + assert.Equal(t, 4, cfg.StreamBufferSize) + assert.Equal(t, "logs", cfg.DataStream.Type) + assert.Equal(t, "akamai.siem", cfg.DataStream.Dataset) + assert.Equal(t, "default", cfg.DataStream.Namespace) +} + +func TestConfigValidate_DataStreamRequired(t *testing.T) { + cfg := validConfig() + cfg.DataStream.Type = "" + assert.ErrorContains(t, cfg.Validate(), "data_stream.type is required") + + cfg = validConfig() + cfg.DataStream.Dataset = "" + assert.ErrorContains(t, cfg.Validate(), "data_stream.dataset is required") + + cfg = validConfig() + cfg.DataStream.Namespace = "" + assert.ErrorContains(t, cfg.Validate(), "data_stream.namespace is required") +} + +func TestConfigValidate_InvalidBatchSize(t *testing.T) { + cfg := validConfig() + cfg.BatchSize = 0 + assert.ErrorContains(t, cfg.Validate(), "batch_size must be greater than 0") +} + +func TestConfigValidate_InvalidStreamBufferSize(t *testing.T) { + cfg := validConfig() + cfg.StreamBufferSize = 0 + assert.ErrorContains(t, cfg.Validate(), "stream_buffer_size must be greater than 0") +} + +func TestConfigValidate_RejectsHTTPAuth(t *testing.T) { + cfg := validConfig() + authID := component.NewID(component.MustNewType("oauth2client")) + cfg.HTTP.Auth = configoptional.Some(configauth.Config{AuthenticatorID: authID}) + assert.ErrorContains(t, cfg.Validate(), "http.auth is not supported") +} diff --git a/receiver/akamaisiemreceiver/doc.go b/receiver/akamaisiemreceiver/doc.go new file mode 100644 index 000000000..a82f99273 --- /dev/null +++ b/receiver/akamaisiemreceiver/doc.go @@ -0,0 +1,21 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:generate mdatagen metadata.yaml + +// Package akamaisiemreceiver provides a receiver that collects logs from Akamai SIEM and sends them to the OpenTelemetry Collector. +package akamaisiemreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver" diff --git a/receiver/akamaisiemreceiver/documentation.md b/receiver/akamaisiemreceiver/documentation.md new file mode 100644 index 000000000..6629ec761 --- /dev/null +++ b/receiver/akamaisiemreceiver/documentation.md @@ -0,0 +1,135 @@ +[comment]: <> (Code generated by mdatagen. DO NOT EDIT.) + +# akamai_siem + +## Internal Telemetry + +The following telemetry is emitted by this component. + +### otelcol_akamai_siem.bytes_received + +Total bytes received from the Akamai SIEM API + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| By | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.cursor_persists + +Total successful cursor persist operations + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {persists} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.events_emitted + +Total events successfully forwarded to the downstream consumer via ConsumeLogs + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {events} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.events_per_page + +Number of events received per API response page + +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| {events} | Histogram | Int | Alpha | + +### otelcol_akamai_siem.events_per_second + +Events per second throughput per poll cycle + +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| {events}/s | Histogram | Double | Alpha | + +### otelcol_akamai_siem.events_received + +Total events received from the Akamai SIEM API + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {events} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.invalid_timestamp_retries + +Total HMAC timestamp retries + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {retries} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.offset_expired + +Total 416 offset out of range errors + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {errors} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.offset_ttl_drops + +Total proactive offset TTL expirations + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {drops} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.page_processing_time + +Time to process a page (NDJSON parse + body-map construction + ConsumeLogs) + +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| s | Histogram | Double | Alpha | + +### otelcol_akamai_siem.pages_processed + +Total API response pages processed + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {pages} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.poll_duration + +Duration of a full poll iteration (may include multiple pages) + +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| s | Histogram | Double | Alpha | + +### otelcol_akamai_siem.recovery_attempts + +Total recovery actions taken (416 replays, timestamp retries, from clamps) + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {attempts} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.request_duration + +Duration of API requests to the Akamai SIEM API + +| Unit | Metric Type | Value Type | Stability | +| ---- | ----------- | ---------- | --------- | +| s | Histogram | Double | Alpha | + +### otelcol_akamai_siem.request_errors + +Total failed API requests (non-200 responses) + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {requests} | Sum | Int | true | Alpha | + +### otelcol_akamai_siem.requests + +Total API requests made to the Akamai SIEM API + +| Unit | Metric Type | Value Type | Monotonic | Stability | +| ---- | ----------- | ---------- | --------- | --------- | +| {requests} | Sum | Int | true | Alpha | diff --git a/receiver/akamaisiemreceiver/factory.go b/receiver/akamaisiemreceiver/factory.go new file mode 100644 index 000000000..201475037 --- /dev/null +++ b/receiver/akamaisiemreceiver/factory.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver" + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/metadata" +) + +// NewFactory creates a new factory for the Akamai SIEM receiver. +func NewFactory() receiver.Factory { + return receiver.NewFactory( + metadata.Type, + createDefaultConfig, + receiver.WithLogs(createLogsReceiver, metadata.LogsStability), + ) +} + +func createLogsReceiver( + _ context.Context, + set receiver.Settings, + cfg component.Config, + cons consumer.Logs, +) (receiver.Logs, error) { + return newAkamaiReceiver(cfg.(*Config), set, cons) +} diff --git a/receiver/akamaisiemreceiver/generated_component_test.go b/receiver/akamaisiemreceiver/generated_component_test.go new file mode 100644 index 000000000..bdc5ff48d --- /dev/null +++ b/receiver/akamaisiemreceiver/generated_component_test.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package akamaisiemreceiver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +var typ = component.MustNewType("akamai_siem") + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, typ, NewFactory().Type()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) + name string + }{ + + { + name: "logs", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogs(ctx, set, cfg, consumertest.NewNop()) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + firstRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + host := newMdatagenNopHost() + require.NoError(t, err) + require.NoError(t, firstRcvr.Start(context.Background(), host)) + require.NoError(t, firstRcvr.Shutdown(context.Background())) + secondRcvr, err := tt.createFn(context.Background(), receivertest.NewNopSettings(typ), cfg) + require.NoError(t, err) + require.NoError(t, secondRcvr.Start(context.Background(), host)) + require.NoError(t, secondRcvr.Shutdown(context.Background())) + }) + } +} + +var _ component.Host = (*mdatagenNopHost)(nil) + +type mdatagenNopHost struct{} + +func newMdatagenNopHost() component.Host { + return &mdatagenNopHost{} +} + +func (mnh *mdatagenNopHost) GetExtensions() map[component.ID]component.Component { + return nil +} + +func (mnh *mdatagenNopHost) GetFactory(_ component.Kind, _ component.Type) component.Factory { + return nil +} diff --git a/receiver/akamaisiemreceiver/generated_package_test.go b/receiver/akamaisiemreceiver/generated_package_test.go new file mode 100644 index 000000000..41ad153a6 --- /dev/null +++ b/receiver/akamaisiemreceiver/generated_package_test.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package akamaisiemreceiver + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/receiver/akamaisiemreceiver/go.mod b/receiver/akamaisiemreceiver/go.mod new file mode 100644 index 000000000..c7ab70a99 --- /dev/null +++ b/receiver/akamaisiemreceiver/go.mod @@ -0,0 +1,84 @@ +module github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver + +go 1.25.0 + +require ( + github.com/google/uuid v1.6.0 + github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/collector/component v1.57.0 + go.opentelemetry.io/collector/component/componenttest v0.151.0 + go.opentelemetry.io/collector/config/configauth v1.57.0 + go.opentelemetry.io/collector/config/confighttp v0.151.0 + go.opentelemetry.io/collector/config/configopaque v1.57.0 + go.opentelemetry.io/collector/config/configoptional v1.57.0 + go.opentelemetry.io/collector/confmap v1.57.0 + go.opentelemetry.io/collector/consumer v1.57.0 + go.opentelemetry.io/collector/consumer/consumertest v0.151.0 + go.opentelemetry.io/collector/extension/xextension v0.151.0 + go.opentelemetry.io/collector/pdata v1.57.0 + go.opentelemetry.io/collector/receiver v1.57.0 + go.opentelemetry.io/collector/receiver/receivertest v0.151.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.28.0 +) + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/knadh/koanf/providers/confmap v1.0.0 // indirect + github.com/knadh/koanf/v2 v2.3.4 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/cors v1.11.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/client v1.57.0 // indirect + go.opentelemetry.io/collector/config/configcompression v1.57.0 // indirect + go.opentelemetry.io/collector/config/configmiddleware v1.57.0 // indirect + go.opentelemetry.io/collector/config/confignet v1.57.0 // indirect + go.opentelemetry.io/collector/config/configtls v1.57.0 // indirect + go.opentelemetry.io/collector/confmap/xconfmap v0.151.0 // indirect + go.opentelemetry.io/collector/consumer/consumererror v0.151.0 // indirect + go.opentelemetry.io/collector/consumer/xconsumer v0.151.0 // indirect + go.opentelemetry.io/collector/extension v1.57.0 // indirect + go.opentelemetry.io/collector/extension/extensionauth v1.57.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v0.151.0 // indirect + go.opentelemetry.io/collector/featuregate v1.57.0 // indirect + go.opentelemetry.io/collector/internal/componentalias v0.151.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.151.0 // indirect + go.opentelemetry.io/collector/pipeline v1.57.0 // indirect + go.opentelemetry.io/collector/receiver/xreceiver v0.151.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/receiver/akamaisiemreceiver/go.sum b/receiver/akamaisiemreceiver/go.sum new file mode 100644 index 000000000..cfa92d556 --- /dev/null +++ b/receiver/akamaisiemreceiver/go.sum @@ -0,0 +1,188 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f h1:RJ+BDPLSHQO7cSjKBqjPJSbi1qfk9WcsjQDtZiw3dZw= +github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f/go.mod h1:VHbbch/X4roIY22jL1s3qRbZhCiRIgUAF/PdSUcx2io= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.4.7 h1:J3ycC8umYxM9A4eF73EofRZu4BxY0jjQnUnkhIBbvws= +github.com/google/go-tpm-tools v0.4.7/go.mod h1:gSyXTZHe3fgbzb6WEGd90QucmsnT1SRdlye82gH8QjQ= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= +github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= +github.com/knadh/koanf/v2 v2.3.4 h1:fnynNSDlujWE+v83hAp8wKr/cdoxHLO0629SN+U8Urc= +github.com/knadh/koanf/v2 v2.3.4/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/collector/client v1.57.0 h1:B6yTRsjQcFa1mqR9EA4sEvcKY5x0QtUFrf8/senhUR8= +go.opentelemetry.io/collector/client v1.57.0/go.mod h1:gdJxZb5MnsJ3WrV6Oy72keeLLeAchv3Wl1gxUDu1WUE= +go.opentelemetry.io/collector/component v1.57.0 h1:WKIqx2Bs0JaAZxDEhsLradXpYxnwAxVFzWhQUmu2q3w= +go.opentelemetry.io/collector/component v1.57.0/go.mod h1:rXLy5mV78e7Gqp/dzFB+nbAFSEuJCipJfp8LbkrvOMg= +go.opentelemetry.io/collector/component/componenttest v0.151.0 h1:0rYcx913VAfD1VyVA9MKPjTrdinUaJGEaOhom8MX5zY= +go.opentelemetry.io/collector/component/componenttest v0.151.0/go.mod h1:vmhG58+J9QHOHaNu8LUD5d13LqldvkzI2jil4+lk+x0= +go.opentelemetry.io/collector/config/configauth v1.57.0 h1:wNm4CYIo3n8zFfk8mW3lppI5CtRqg7SGwfO+k20iaiY= +go.opentelemetry.io/collector/config/configauth v1.57.0/go.mod h1:Tzoa6JoVmQ6mhgSeYf0mqQk8GGC77Ilj7ahPko8JNZQ= +go.opentelemetry.io/collector/config/configcompression v1.57.0 h1:i8GgiuhJ2EdBfcwjx3RaScQEyIN0NFZPdqrxx6zRstY= +go.opentelemetry.io/collector/config/configcompression v1.57.0/go.mod h1:SEcE2uFLHHPc/Vi8WCkW5MhOMUwaT321HBdZ3P8x8D0= +go.opentelemetry.io/collector/config/confighttp v0.151.0 h1:9cur4OryZGpIgAuBGtN8oTIBr/CNBsuMqoexZUK6yok= +go.opentelemetry.io/collector/config/confighttp v0.151.0/go.mod h1:edcQviVMlQlFCKjdfZdqk7NfgjgXsErEZeNC6lq0r54= +go.opentelemetry.io/collector/config/configmiddleware v1.57.0 h1:7RVS4DHAbMHDlsLDlNDDbaDolYxQOPOYwqJovbBAfog= +go.opentelemetry.io/collector/config/configmiddleware v1.57.0/go.mod h1:ia49Ny/3Ew6SeQc+ZYMewjyGr8IPg9y8XVSgbmVqpQE= +go.opentelemetry.io/collector/config/confignet v1.57.0 h1:X2tVVQiKo4/V/A1/4GzKPipZsdvrQdgZCfCOHIW6jo8= +go.opentelemetry.io/collector/config/confignet v1.57.0/go.mod h1:Op+r1B/DtzXgIuKEL7/JkTqtJdL9veu2uEXvSxH3lks= +go.opentelemetry.io/collector/config/configopaque v1.57.0 h1:cV56mFftZyU3vaLus77/mHboRzJ2GqzOHIsyO0zvCuw= +go.opentelemetry.io/collector/config/configopaque v1.57.0/go.mod h1:6/DgOtshwnuiTehze+czXeN4GgKgR3ZD+wxBumlgUWM= +go.opentelemetry.io/collector/config/configoptional v1.57.0 h1:eN/MCyDJINsMR3KSOJ+3q4rOdEm4lzb04sRD1iG4pUE= +go.opentelemetry.io/collector/config/configoptional v1.57.0/go.mod h1:jVagnW+2jTO9WQiD17wK7VqbZ+nMzvfoNlHUi0WdhpI= +go.opentelemetry.io/collector/config/configtls v1.57.0 h1:/xyZxadD/Kn9X0V1JFQzaEisuA8DXmWTbsb7BOvE+eE= +go.opentelemetry.io/collector/config/configtls v1.57.0/go.mod h1:PVY1IqNQIpxRmON1Uw0P1n+IjhWOAGPamN/VVQtatMY= +go.opentelemetry.io/collector/confmap v1.57.0 h1:5AuK920dJmV8zxQAiODi2JHPl2r1HmEHHMaBSC+qF5I= +go.opentelemetry.io/collector/confmap v1.57.0/go.mod h1:ifmog4kqEMM037qX04qEbom5CcxhmkadLUqhi2Vkuec= +go.opentelemetry.io/collector/confmap/xconfmap v0.151.0 h1:txpp8lH/J2sKsQXEmV0TXTIrDS7n0Bo2bPJR+mhcP3M= +go.opentelemetry.io/collector/confmap/xconfmap v0.151.0/go.mod h1:3R0Ru3Gsz6HKzjMecZPlTFDzFxaxAOl23ptm+xlsAA0= +go.opentelemetry.io/collector/consumer v1.57.0 h1:jyDh4GkYPuIXNB0UJIh33NAzZoTCVNkwS+XWdlI08P8= +go.opentelemetry.io/collector/consumer v1.57.0/go.mod h1:tJKbog9Xw/8y66aWd/C+21BMuQkOWn/lF4bzJDRC9OM= +go.opentelemetry.io/collector/consumer/consumererror v0.151.0 h1:tspiI5WE/xwYNAzR61I1XhmNYhSXdsR0Yf1Dk2MZIfQ= +go.opentelemetry.io/collector/consumer/consumererror v0.151.0/go.mod h1:dxnhPRnw3kjyweDy9pGWUNVueem15kxlvqRp0qY5k6U= +go.opentelemetry.io/collector/consumer/consumertest v0.151.0 h1:qByIVlFh9RAR/newAk/sN5i1zoIXKa2K1hRNVfye8LU= +go.opentelemetry.io/collector/consumer/consumertest v0.151.0/go.mod h1:eAGCGxkq+aABLmlr3PvMOqz3ZJbmn/lUqCbbffabSi4= +go.opentelemetry.io/collector/consumer/xconsumer v0.151.0 h1:eKIYxuPBEIrjZMAkyKBUWrlpHAE9OgxXBjq7PMSeXkE= +go.opentelemetry.io/collector/consumer/xconsumer v0.151.0/go.mod h1:9K97TkCN7XYfwKzPzktozrWc3Qw/4A1T4XgMn9TnG0c= +go.opentelemetry.io/collector/extension v1.57.0 h1:xrKqf2CK8AjEJFtxky84l7PkzbDrFv5jomfsRDgeW80= +go.opentelemetry.io/collector/extension v1.57.0/go.mod h1:jwIanPruVtNwWbkXOi8ikfWj0mIl4m7vZdGQPDvUJcE= +go.opentelemetry.io/collector/extension/extensionauth v1.57.0 h1:SUTbKc3sVFrCOQcU8NoNsVySKvmIOosn4z3H5tyg9FQ= +go.opentelemetry.io/collector/extension/extensionauth v1.57.0/go.mod h1:iXhR9e5eC2XbdDf/Z17QJIV+wQx1E5DTth2oE3MmVMA= +go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.151.0 h1:hBlyGreyBrKQLXrSCoClkgeryyNe388LuZXrBHjwmc4= +go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.151.0/go.mod h1:jn1J+XxwNrsdEarKB3CpCLezuTLQipzLozFTeFQvFuk= +go.opentelemetry.io/collector/extension/extensionmiddleware v0.151.0 h1:UxqZPlF345Xnrl8fJWWpJEDNkKbiuu37r0XlfOPkxzc= +go.opentelemetry.io/collector/extension/extensionmiddleware v0.151.0/go.mod h1:IzeOB7CZmf/92KGu4Sm6mODu5tejgupcs1tW2eAkXmY= +go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.151.0 h1:N8btWjsACekA8mqD+B+So3aqHF2X7X/MWIBJS0PpYXM= +go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.151.0/go.mod h1:HYRwLtjSJDeCsJg8ifXjFeAnOb1CnB8il6Z2Cilrc9k= +go.opentelemetry.io/collector/extension/xextension v0.151.0 h1:N8Bs5cPUpnpQDKVBkCGLRHuLmLubkI1BANl/tGp93Qc= +go.opentelemetry.io/collector/extension/xextension v0.151.0/go.mod h1:DXK1pzPdH5zQpKfTYW7eFBUf3ES/RkLxJUnSxco/ZrA= +go.opentelemetry.io/collector/featuregate v1.57.0 h1:KPDSUKYn6MHwgyGRSGPPcW/G96HH93pxuvvPwM+R8nY= +go.opentelemetry.io/collector/featuregate v1.57.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU= +go.opentelemetry.io/collector/internal/componentalias v0.151.0 h1:5IJn4XXRbjGrJCuIByHzxgHqwC0Hcl99tM+PoyYzjJY= +go.opentelemetry.io/collector/internal/componentalias v0.151.0/go.mod h1:c70sQuXHQZWSYCyc0y/VynqJdmEeBunSmEy3xfLQPWE= +go.opentelemetry.io/collector/internal/testutil v0.151.0 h1:CFjDItLuqzblItOsnK6IPSdrsOaZCaDjYpB8qWG+XHI= +go.opentelemetry.io/collector/internal/testutil v0.151.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE= +go.opentelemetry.io/collector/pdata v1.57.0 h1:oDWBMjEIqyJO3GJEB+iwqxj47rxDK19OKzwaFEaE4sg= +go.opentelemetry.io/collector/pdata v1.57.0/go.mod h1:wZojinP6mNhLXudH8QXx/bjWzOsKMxi/FXwnk+12G/w= +go.opentelemetry.io/collector/pdata/pprofile v0.151.0 h1:hsU0+DpkvhJh3xL1Y8CX2vAPdLMoJLiw+C+rAMsaxZc= +go.opentelemetry.io/collector/pdata/pprofile v0.151.0/go.mod h1:5zfGTQqRuaKyh2SRaZi4SV4nSD8TzY1kYoOjniOD3uk= +go.opentelemetry.io/collector/pdata/testdata v0.151.0 h1:ye09e8UMADdVrQjLgCznZxmM8ra7ciAuOCteHDzgHjc= +go.opentelemetry.io/collector/pdata/testdata v0.151.0/go.mod h1:h5+Ys9F+pf64cGt5cZCDtRsrkOnvjgpcONr8pFA3KBc= +go.opentelemetry.io/collector/pipeline v1.57.0 h1:nlevGN75Vt/Fp0HTaDjZpUHQf5QFA6o2asSmzSoBVkA= +go.opentelemetry.io/collector/pipeline v1.57.0/go.mod h1:RD90NG3Jbk965Xaqym3JyHkuol4uZJjQVUkD9ddXJIs= +go.opentelemetry.io/collector/receiver v1.57.0 h1:Aq8hcLByUOOrouekGPsxvaAHbUMxa4NM2Ok84KHjdcE= +go.opentelemetry.io/collector/receiver v1.57.0/go.mod h1:6y2UO4pmiT85z9JApUmRiP/7yX7zY5cmxbMgXvvMOA0= +go.opentelemetry.io/collector/receiver/receivertest v0.151.0 h1:U1iALsnbrAM3QdT4977y/S+Tbe68iS42qoeY5zgGXfE= +go.opentelemetry.io/collector/receiver/receivertest v0.151.0/go.mod h1:MBAEOfep8HMpdOcD5j1LOpP/GTjqz5hmZijGyYGxkGE= +go.opentelemetry.io/collector/receiver/xreceiver v0.151.0 h1:Ib86H9rBBD76wmpKQFRlwYqugvxbDtq8Lg6bVROBw4k= +go.opentelemetry.io/collector/receiver/xreceiver v0.151.0/go.mod h1:9Jqb3YKmgkZeLmHx5QBLBgdoLiu2T9q4B+o6LGTR/8U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM= +go.opentelemetry.io/proto/slim/otlp v1.10.0/go.mod h1:lV9250stpjYLPNA5viFabIgP2QlUGRT1GdTgAf8SIUk= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:RUF5rO0hAlgiJt1fzQVzcVs3vZVNHIcMLgOgG4rWNcQ= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/receiver/akamaisiemreceiver/img/architecture_overview.svg b/receiver/akamaisiemreceiver/img/architecture_overview.svg new file mode 100644 index 000000000..7e3faa99c --- /dev/null +++ b/receiver/akamaisiemreceiver/img/architecture_overview.svg @@ -0,0 +1,128 @@ + + + + + + + + + Akamai SIEM Receiver — Architecture Overview + + + + AKAMAI CLOUD + + SIEM API + /siem/v1/configs/{id} + + EdgeGrid HMAC-SHA256 + + + + akamai_siem receiver + + + Poller — Chain State Machine + 3-branch: offset drain | chain replay | new chain + + + NDJSON Parser + bounded channel streaming + + + Cursor Store + file-based, atomic write + + + Telemetry + 19 OTel metrics + tracing + + + confighttp.ClientConfig + TLS, proxy, timeout + + + + bodymap output (single format) + + + + plog.Logs shape + body map: {message, data_stream.{type,dataset,namespace}} + resource: data_stream.{type,dataset,namespace} + scope: elastic.mapping.mode: bodymap + → Elasticsearch (bodymap) + Kibana + + + + OUTPUT: plog.Logs → ConsumeLogs() + + + Cursor, chain drain, offset recovery, telemetry — identical in all modes + + + + Batch + (optional) + + + Elasticsearch + Exporter + + + + ELASTICSEARCH (ECS path) + + + Index Template: logs-akamai.siem-* + sets default_pipeline on match + + + Ingest Pipeline (triggered) + JSON parse | ECS map | GeoIP | base64 decode | _id fingerprint + + + Data Stream: logs-akamai.siem-default + enriched ECS documents stored + + + installed by Akamai integration in Kibana (Fleet) + + + + + + + + Kibana + Akamai Dashboard + + + + + NDJSON + + + signed GET + + + + + + + + + + ECS + + + + + + + + + Body map {message: rawJSON, data_stream.*}; resource data_stream.*; scope elastic.mapping.mode: bodymap + ES exporter writes body map fields directly into the document; ingest pipeline enriches; Kibana dashboards work + Cursor persistence via OTel storage extension (e.g. file_storage) for safe resume across restarts + diff --git a/receiver/akamaisiemreceiver/img/architecture_raw.svg b/receiver/akamaisiemreceiver/img/architecture_raw.svg new file mode 100644 index 000000000..36fd9f902 --- /dev/null +++ b/receiver/akamaisiemreceiver/img/architecture_raw.svg @@ -0,0 +1,72 @@ + + + + + + + + + Akamai SIEM Receiver — bodymap to Elasticsearch + + + + SIEM API + NDJSON stream + + + + + + + akamai_siem receiver + Body map: {message, data_stream.*} + resource: data_stream.{type,dataset,namespace} + scope: elastic.mapping.mode: bodymap + ~0.009ms/event | 1.4 KB/event + + + + + + + Batch + (optional) + + + + + + + ES Exporter + Body → "message" + auto ECS mode + + + + + + + Elasticsearch + Index: logs-akamai.siem-* + Ingest pipeline: JSON parse + → ECS mapping → GeoIP + → Kibana dashboard ready + + + + Best For + Elasticsearch + Kibana deployments + Fastest mode: ~0.009ms CPU, 1.4 KB per event + Lowest memory footprint: ~105 MB RSS + Zero parsing — raw JSON passthrough to ingest pipeline + ES exporter auto-detects ECS mode via context metadata + Pre-built Kibana dashboards work out of the box + + + + Trade-offs + Backend must support ingest pipeline processing + Akamai integration assets must be installed + Non-pipeline backends receive raw JSON body + Events not queryable by OTel Collector processors + diff --git a/receiver/akamaisiemreceiver/integration_test.go b/receiver/akamaisiemreceiver/integration_test.go new file mode 100644 index 000000000..b94ece2b9 --- /dev/null +++ b/receiver/akamaisiemreceiver/integration_test.go @@ -0,0 +1,167 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +// Integration tests verify the exact output of the receiver against realistic +// Akamai SIEM API responses using a local httptest server. +// Test data: testdata/siem_response_full.ndjson — 3 events with different attack +// types, HTTP methods, geo locations, and applied actions. + +func TestIntegration_FullResponse(t *testing.T) { + ndjson, err := os.ReadFile("testdata/siem_response_full.ndjson") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(ndjson) + })) + defer server.Close() + + sink := &consumertest.LogsSink{} + rcv := createTestReceiver(t, server.URL, sink) + + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + assert.Eventually(t, func() bool { return sink.LogRecordCount() >= 3 }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + + records := collectRecords(allLogs) + require.Len(t, records, 3, "expected 3 events from test data") + + for i, lr := range records { + body := intBodyMessage(t, lr) + assert.NotEmpty(t, body, "record %d body should not be empty", i) + assert.Contains(t, body, `"attackData"`, "record %d should contain raw Akamai JSON", i) + assert.Contains(t, body, `"httpMessage"`, "record %d should contain httpMessage", i) + assert.Equal(t, 0, lr.Attributes().Len(), "record %d should have no record attributes, got %d", i, lr.Attributes().Len()) + } + + assert.Contains(t, intBodyMessage(t, records[0]), `"appliedAction":"deny"`) + assert.Contains(t, intBodyMessage(t, records[0]), `"method":"POST"`) + assert.Contains(t, intBodyMessage(t, records[0]), `"host":"api.example.com"`) + + assert.Contains(t, intBodyMessage(t, records[1]), `"appliedAction":"monitor"`) + assert.Contains(t, intBodyMessage(t, records[1]), `"method":"GET"`) + assert.Contains(t, intBodyMessage(t, records[1]), `"country":"BR"`) + + assert.Contains(t, intBodyMessage(t, records[2]), `"appliedAction":"alert"`) + assert.Contains(t, intBodyMessage(t, records[2]), `"method":"DELETE"`) + assert.Contains(t, intBodyMessage(t, records[2]), `"status":"500"`) +} + +func TestIntegration_EmptyResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"offset":"empty-cursor","total":0,"limit":10000}`) + })) + defer server.Close() + + sink := &consumertest.LogsSink{} + rcv := createTestReceiver(t, server.URL, sink) + + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + time.Sleep(2 * time.Second) + require.NoError(t, rcv.Shutdown(context.Background())) + + assert.Equal(t, 0, sink.LogRecordCount()) +} + +func TestIntegration_SeverityUnset(t *testing.T) { + // The receiver does not interpret event content — severity stays unspecified. + ndjson, err := os.ReadFile("testdata/siem_response_full.ndjson") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(ndjson) + })) + defer server.Close() + + sink := &consumertest.LogsSink{} + rcv := createTestReceiver(t, server.URL, sink) + + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + assert.Eventually(t, func() bool { return sink.LogRecordCount() >= 3 }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + records := collectRecords(sink.AllLogs()) + for i, lr := range records { + assert.Equal(t, plog.SeverityNumberUnspecified, lr.SeverityNumber(), "record %d should have unspecified severity", i) + } +} + +// --- helpers --- + +func createTestReceiver(t *testing.T, serverURL string, sink *consumertest.LogsSink) *akamaiReceiver { + t.Helper() + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = serverURL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), ClientSecret: configopaque.String("cs"), AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + + set := receivertest.NewNopSettings(NewFactory().Type()) + rcv, err := newAkamaiReceiver(cfg, set, sink) + require.NoError(t, err) + return rcv +} + +func collectRecords(allLogs []plog.Logs) []plog.LogRecord { + var records []plog.LogRecord + for _, logs := range allLogs { + for i := 0; i < logs.ResourceLogs().Len(); i++ { + rl := logs.ResourceLogs().At(i) + for j := 0; j < rl.ScopeLogs().Len(); j++ { + sl := rl.ScopeLogs().At(j) + for k := 0; k < sl.LogRecords().Len(); k++ { + records = append(records, sl.LogRecords().At(k)) + } + } + } + } + return records +} + +// intBodyMessage extracts the "message" string from a LogRecord body map. +func intBodyMessage(t *testing.T, lr plog.LogRecord) string { + t.Helper() + require.Equal(t, pcommon.ValueTypeMap, lr.Body().Type(), "body should be a map") + v, ok := lr.Body().Map().Get("message") + require.True(t, ok, "body map should have 'message' key") + return v.Str() +} diff --git a/receiver/akamaisiemreceiver/internal/akamaiclient/benchmark_test.go b/receiver/akamaisiemreceiver/internal/akamaiclient/benchmark_test.go new file mode 100644 index 000000000..7cabb649f --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/akamaiclient/benchmark_test.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaiclient + +import ( + "context" + "fmt" + "strings" + "testing" +) + +const benchEvent = `{"attackData":{"appliedAction":"tarpit","clientIP":"198.51.100.1","configId":"67217","policyId":"PNWD_110088"},"httpMessage":{"bytes":"0","host":"example.com","method":"GET","path":"/api/test","port":"443","protocol":"HTTP/1.1","query":"q=test","requestId":"f3fe4c34","start":"1762365006","status":"200","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"}` + +func BenchmarkStreamEvents_10(b *testing.B) { benchStreamEvents(b, 10) } +func BenchmarkStreamEvents_100(b *testing.B) { benchStreamEvents(b, 100) } +func BenchmarkStreamEvents_1000(b *testing.B) { benchStreamEvents(b, 1000) } +func BenchmarkStreamEvents_10000(b *testing.B) { benchStreamEvents(b, 10000) } +func BenchmarkStreamEvents_100000(b *testing.B) { benchStreamEvents(b, 100000) } + +func benchStreamEvents(b *testing.B, n int) { + b.Helper() + body := buildBody(n) + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + reader := strings.NewReader(body) + ch := make(chan string, 100) + var count int + go func() { + defer close(ch) + _, count, _ = StreamEvents(ctx, reader, ch) + }() + // Drain channel. + received := 0 + for range ch { + received++ + } + if received != n || count != n { + b.Fatalf("expected %d events, got received=%d count=%d", n, received, count) + } + } + b.ReportMetric(float64(n), "events/op") +} + +func buildBody(n int) string { + var sb strings.Builder + for i := 0; i < n; i++ { + sb.WriteString(benchEvent) + sb.WriteString("\n") + } + sb.WriteString(fmt.Sprintf(`{"offset":"bench-cursor","total":%d,"limit":%d}`, n, n+1)) + sb.WriteString("\n") + return sb.String() +} diff --git a/receiver/akamaisiemreceiver/internal/akamaiclient/client.go b/receiver/akamaisiemreceiver/internal/akamaiclient/client.go new file mode 100644 index 000000000..077e481fa --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/akamaiclient/client.go @@ -0,0 +1,281 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package akamaiclient contains the core Akamai SIEM API client and NDJSON +// streaming parser. +package akamaiclient // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/akamaiclient" + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "go.uber.org/zap" +) + +const siemAPIPath = "/siem/v1/configs/" + +// APIError represents a non-200 response from the Akamai SIEM API. +// +// Detail is parsed from the JSON `detail` field per RFC 7807 problem-details +// and is the value matched by IsInvalidTimestamp / IsFromTooOld to drive +// recovery decisions — keep it strictly structured. +// +// Body is the raw response body, useful for debugging when the response is +// not JSON (HTML error page, proxy error, etc.). It is logged at debug level +// from the construction site and never used for state-machine decisions. +type APIError struct { + StatusCode int + Status string + Detail string + Body string +} + +func (e *APIError) Error() string { + if e.Detail != "" { + return fmt.Sprintf("akamai API error: %s (%d): %s", e.Status, e.StatusCode, e.Detail) + } + return fmt.Sprintf("akamai API error: %s (%d)", e.Status, e.StatusCode) +} + +// IsInvalidTimestamp returns true if the error indicates an invalid HMAC timestamp. +func (e *APIError) IsInvalidTimestamp() bool { + return e.StatusCode == 400 && strings.Contains(strings.ToLower(e.Detail), "invalid timestamp") +} + +// IsOffsetOutOfRange returns true if the offset is expired (416). +func (e *APIError) IsOffsetOutOfRange() bool { + return e.StatusCode == 416 +} + +// IsFromTooOld returns true if the from parameter exceeds the max lookback. +func (e *APIError) IsFromTooOld() bool { + if e.StatusCode != 400 { + return false + } + lower := strings.ToLower(e.Detail) + return strings.Contains(lower, "out of range") || strings.Contains(lower, "too old") +} + +// FetchParams contains parameters for an API fetch request. +type FetchParams struct { + Offset string + From int64 + To int64 + Limit int +} + +// FetchMode returns "offset" or "time" based on the params. +func (p FetchParams) FetchMode() string { + if p.Offset != "" { + return "offset" + } + return "time" +} + +// OffsetContext is the pagination metadata returned as the last line in +// NDJSON responses. A valid offset context has Limit > 0. The API also +// returns a "total" field which the receiver does not use. +type OffsetContext struct { + Offset string `json:"offset"` + Limit int `json:"limit"` +} + +// Client is the Akamai SIEM API HTTP client. +type Client struct { + httpClient *http.Client + baseURL *url.URL + configIDs string + log *zap.Logger + lastContentLength int64 +} + +// NewClient creates a new Akamai SIEM API client. The httpClient should +// already have EdgeGrid signing configured on its transport. +func NewClient(httpClient *http.Client, endpoint, configIDs string, log *zap.Logger) (*Client, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + return &Client{ + httpClient: httpClient, + baseURL: u, + configIDs: configIDs, + log: log.Named("client"), + }, nil +} + +// FetchResponse makes the HTTP request and returns the response body. +// On non-200 status, the body is consumed and an *APIError is returned. +// The caller must close the returned body on success. +func (c *Client) FetchResponse(ctx context.Context, params FetchParams) (io.ReadCloser, error) { + reqURL := c.buildRequestURL(params) + c.log.Debug("fetching events", + zap.String("mode", params.FetchMode()), + zap.Int("limit", params.Limit), + zap.String("offset", params.Offset), + zap.Int64("from", params.From), + zap.Int64("to", params.To), + ) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + c.log.Error("HTTP request failed", + zap.Error(err), + zap.String("url", reqURL), + zap.String("mode", params.FetchMode()), + ) + return nil, fmt.Errorf("request failed: %w", err) + } + + c.log.Debug("API response received", + zap.Int("status_code", resp.StatusCode), + zap.String("mode", params.FetchMode()), + ) + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + apiErr := &APIError{ + StatusCode: resp.StatusCode, + Status: resp.Status, + Body: string(body), + } + if len(body) > 0 { + var errResp struct { + Detail string `json:"detail"` + } + if json.Unmarshal(body, &errResp) == nil { + apiErr.Detail = errResp.Detail + } + // Log the raw body at debug level so non-JSON responses (HTML error + // pages, proxy errors) are visible without polluting Detail, which + // is substring-matched for recovery decisions. + c.log.Debug("non-200 response body", + zap.Int("status_code", resp.StatusCode), + zap.ByteString("body", body), + ) + } + return nil, apiErr + } + + c.lastContentLength = resp.ContentLength + return resp.Body, nil +} + +// LastContentLength returns the Content-Length from the most recent successful +// response. Returns -1 if unknown (chunked transfer). +func (c *Client) LastContentLength() int64 { + return c.lastContentLength +} + +func (c *Client) buildRequestURL(params FetchParams) string { + u := *c.baseURL + u.Path = siemAPIPath + c.configIDs + + query := url.Values{} + query.Set("limit", strconv.Itoa(params.Limit)) + + if params.Offset != "" { + query.Set("offset", params.Offset) + } else { + if params.From > 0 { + query.Set("from", strconv.FormatInt(params.From, 10)) + } + if params.To > 0 { + query.Set("to", strconv.FormatInt(params.To, 10)) + } + } + + u.RawQuery = query.Encode() + return u.String() +} + +// StreamEvents reads NDJSON lines from body, pushing event lines into eventCh +// using a one-line delay pattern. The last line is checked for offset context +// metadata and returned separately. +// +// The caller must close eventCh after StreamEvents returns. +func StreamEvents(ctx context.Context, body io.Reader, eventCh chan<- string) (pageCtx OffsetContext, count int, err error) { + scanner := bufio.NewScanner(body) + const maxTokenSize = 10 * 1024 * 1024 // 10MB max line size + scanner.Buffer(make([]byte, 64*1024), maxTokenSize) + + // send pushes s onto eventCh and bumps count, returning ctx.Err() if the + // caller cancelled. + send := func(s string) error { + select { + case eventCh <- s: + count++ + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + + // 1-line delay: hold back the current line until the next is read. + // This lets us check if the last line is offset context without emitting it. + var prev string + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + if prev != "" { + if err = send(prev); err != nil { + return OffsetContext{}, count, err + } + } + prev = line + } + + if err = scanner.Err(); err != nil { + return OffsetContext{}, count, fmt.Errorf("error reading response: %w", err) + } + if prev == "" { + return OffsetContext{}, 0, nil + } + + // Last line: try to unmarshal as offset context. If valid, don't emit it. + if err = json.Unmarshal([]byte(prev), &pageCtx); err == nil && pageCtx.Offset != "" && pageCtx.Limit > 0 { + return pageCtx, count, nil + } + + // Last line was a regular event, not offset context. + if err = send(prev); err != nil { + return OffsetContext{}, count, err + } + return OffsetContext{}, count, nil +} + +// Close releases resources held by the client. +func (c *Client) Close() { + c.httpClient.CloseIdleConnections() +} diff --git a/receiver/akamaisiemreceiver/internal/akamaiclient/client_test.go b/receiver/akamaisiemreceiver/internal/akamaiclient/client_test.go new file mode 100644 index 000000000..2c051d8b4 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/akamaiclient/client_test.go @@ -0,0 +1,276 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaiclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +// collectStreamEvents is a test helper that runs StreamEvents with a channel +// and collects all events into a slice, returning them along with the offset +// context. This mirrors the old slice-based API for test convenience. +func collectStreamEvents(ctx context.Context, body io.Reader) ([]string, OffsetContext, error) { + ch := make(chan string, 1000) + var pageCtx OffsetContext + var count int + var streamErr error + + go func() { + defer close(ch) + pageCtx, count, streamErr = StreamEvents(ctx, body, ch) + }() + + var events []string + for e := range ch { + events = append(events, e) + } + _ = count + return events, pageCtx, streamErr +} + +func TestStreamEvents_WithOffsetContext(t *testing.T) { + body := strings.NewReader( + `{"attackData":{"rule":"950004"},"httpMessage":{"host":"example.com"}}` + "\n" + + `{"attackData":{"rule":"990011"},"httpMessage":{"host":"example.com"}}` + "\n" + + `{"offset":"next-cursor-abc","total":2,"limit":10000}` + "\n", + ) + + events, pageCtx, err := collectStreamEvents(context.Background(), body) + require.NoError(t, err) + assert.Len(t, events, 2) + assert.Equal(t, "next-cursor-abc", pageCtx.Offset) + assert.Equal(t, 10000, pageCtx.Limit) + + for _, e := range events { + assert.True(t, json.Valid([]byte(e)), "event should be valid JSON: %s", e) + } +} + +func TestStreamEvents_WithoutOffsetContext(t *testing.T) { + body := strings.NewReader( + `{"event":"one"}` + "\n" + + `{"event":"two"}` + "\n", + ) + + events, pageCtx, err := collectStreamEvents(context.Background(), body) + require.NoError(t, err) + assert.Len(t, events, 2) + assert.Empty(t, pageCtx.Offset) + assert.Zero(t, pageCtx.Limit) +} + +func TestStreamEvents_EmptyBody(t *testing.T) { + events, pageCtx, err := collectStreamEvents(context.Background(), strings.NewReader("")) + require.NoError(t, err) + assert.Empty(t, events) + assert.Empty(t, pageCtx.Offset) +} + +func TestStreamEvents_OnlyOffsetContext(t *testing.T) { + events, pageCtx, err := collectStreamEvents(context.Background(), + strings.NewReader(`{"offset":"abc","total":0,"limit":10000}`+"\n")) + require.NoError(t, err) + assert.Empty(t, events) + assert.Equal(t, "abc", pageCtx.Offset) +} + +func TestStreamEvents_BlankLines(t *testing.T) { + body := strings.NewReader("\n\n" + `{"event":"one"}` + "\n\n" + `{"offset":"x","total":1,"limit":100}` + "\n\n") + events, pageCtx, err := collectStreamEvents(context.Background(), body) + require.NoError(t, err) + assert.Len(t, events, 1) + assert.Equal(t, "x", pageCtx.Offset) +} + +func TestStreamEvents_ContextCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + // Use a small bounded channel to trigger the context check in the select. + ch := make(chan string, 1) + body := strings.NewReader(`{"event":"one"}` + "\n" + `{"event":"two"}` + "\n") + _, _, err := StreamEvents(ctx, body, ch) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestStreamEvents_BackPressure(t *testing.T) { + // Bounded channel of size 1 — scanner must block when channel is full. + ch := make(chan string, 1) + body := strings.NewReader( + `{"event":"one"}` + "\n" + + `{"event":"two"}` + "\n" + + `{"event":"three"}` + "\n" + + `{"offset":"x","total":3,"limit":100}` + "\n", + ) + + done := make(chan struct{}) + var pageCtx OffsetContext + var count int + go func() { + defer close(done) + pageCtx, count, _ = StreamEvents(context.Background(), body, ch) + close(ch) + }() + + // Drain slowly to exercise back-pressure. + var events []string + for e := range ch { + events = append(events, e) + } + <-done + + assert.Len(t, events, 3) + assert.Equal(t, 3, count) + assert.Equal(t, "x", pageCtx.Offset) +} + +func TestClient_FetchResponse_Success(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/siem/v1/configs/12345", r.URL.Path) + assert.Equal(t, "10000", r.URL.Query().Get("limit")) + assert.Equal(t, "test-offset", r.URL.Query().Get("offset")) + _, _ = fmt.Fprintln(w, `{"event":"data"}`) + _, _ = fmt.Fprintln(w, `{"offset":"next","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := NewClient(server.Client(), server.URL, "12345", log) + require.NoError(t, err) + + body, err := client.FetchResponse(context.Background(), FetchParams{ + Offset: "test-offset", + Limit: 10000, + }) + require.NoError(t, err) + defer func() { _ = body.Close() }() + + events, pageCtx, err := collectStreamEvents(context.Background(), body) + require.NoError(t, err) + assert.Len(t, events, 1) + assert.Equal(t, "next", pageCtx.Offset) +} + +func TestClient_FetchResponse_TimeBased(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "1000", r.URL.Query().Get("from")) + assert.Equal(t, "2000", r.URL.Query().Get("to")) + assert.Empty(t, r.URL.Query().Get("offset")) + w.WriteHeader(200) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := NewClient(server.Client(), server.URL, "99", log) + require.NoError(t, err) + + body, err := client.FetchResponse(context.Background(), FetchParams{ + From: 1000, + To: 2000, + Limit: 100, + }) + require.NoError(t, err) + _ = body.Close() +} + +func TestClient_FetchResponse_APIError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(416) + _, _ = fmt.Fprintln(w, `{"detail":"offset out of range"}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := NewClient(server.Client(), server.URL, "12345", log) + require.NoError(t, err) + + _, err = client.FetchResponse(context.Background(), FetchParams{Offset: "stale", Limit: 100}) + require.Error(t, err) + + var apiErr *APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, 416, apiErr.StatusCode) + assert.True(t, apiErr.IsOffsetOutOfRange()) +} + +func TestClient_FetchResponse_InvalidTimestamp(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(400) + _, _ = fmt.Fprintln(w, `{"detail":"invalid timestamp"}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := NewClient(server.Client(), server.URL, "12345", log) + require.NoError(t, err) + + _, err = client.FetchResponse(context.Background(), FetchParams{Limit: 100}) + require.Error(t, err) + + var apiErr *APIError + require.ErrorAs(t, err, &apiErr) + assert.True(t, apiErr.IsInvalidTimestamp()) +} + +func TestAPIError_Error_NoDetail(t *testing.T) { + err := &APIError{StatusCode: 500, Status: "500 Internal Server Error"} + assert.Equal(t, "akamai API error: 500 Internal Server Error (500)", err.Error()) +} + +func TestAPIError_Error_WithDetail(t *testing.T) { + err := &APIError{StatusCode: 400, Status: "400 Bad Request", Detail: "bad param"} + assert.Contains(t, err.Error(), "bad param") +} + +func TestNewClient_InvalidURL(t *testing.T) { + log := zaptest.NewLogger(t) + _, err := NewClient(&http.Client{}, "://invalid", "1", log) + assert.Error(t, err) +} + +func TestClient_Close(t *testing.T) { + log := zaptest.NewLogger(t) + client, err := NewClient(&http.Client{}, "https://example.com", "1", log) + require.NoError(t, err) + client.Close() // Should not panic +} + +func TestClient_FetchResponse_NetworkError(t *testing.T) { + log := zaptest.NewLogger(t) + client, err := NewClient(&http.Client{}, "https://localhost:1", "1", log) + require.NoError(t, err) + _, err = client.FetchResponse(context.Background(), FetchParams{Limit: 100}) + assert.Error(t, err) +} + +func TestAPIError_IsFromTooOld(t *testing.T) { + assert.True(t, (&APIError{StatusCode: 400, Detail: "from parameter is out of range"}).IsFromTooOld()) + assert.True(t, (&APIError{StatusCode: 400, Detail: "timestamp is too old"}).IsFromTooOld()) + assert.False(t, (&APIError{StatusCode: 400, Detail: "invalid timestamp"}).IsFromTooOld()) + assert.False(t, (&APIError{StatusCode: 416, Detail: "out of range"}).IsFromTooOld()) +} diff --git a/receiver/akamaisiemreceiver/internal/auth/edgegrid.go b/receiver/akamaisiemreceiver/internal/auth/edgegrid.go new file mode 100644 index 000000000..96f2d5b6c --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/auth/edgegrid.go @@ -0,0 +1,118 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package auth implements Akamai EdgeGrid HMAC-SHA256 request signing. +package auth // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/auth" + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "fmt" + "net/http" + "strings" + "time" + + "github.com/google/uuid" +) + +// EdgeGridSigner signs HTTP requests using Akamai EdgeGrid authentication. +type EdgeGridSigner struct { + clientToken string + clientSecret string + accessToken string +} + +// NewEdgeGridSigner creates a new signer with the provided credentials. +func NewEdgeGridSigner(clientToken, clientSecret, accessToken string) *EdgeGridSigner { + return &EdgeGridSigner{ + clientToken: clientToken, + clientSecret: clientSecret, + accessToken: accessToken, + } +} + +// Sign adds the EdgeGrid authorization header to the request. +func (s *EdgeGridSigner) Sign(req *http.Request) { + timestamp := time.Now().UTC().Format("20060102T15:04:05-0700") + nonce := uuid.New().String() + + authBase := fmt.Sprintf( + "EG1-HMAC-SHA256 client_token=%s;access_token=%s;timestamp=%s;nonce=%s;", + s.clientToken, s.accessToken, timestamp, nonce, + ) + + signingKey := createSigningKey(timestamp, s.clientSecret) + dataToSign := buildDataToSign(req, authBase) + signature := computeSignature(dataToSign, signingKey) + + req.Header.Set("Authorization", authBase+"signature="+signature) +} + +func createSigningKey(timestamp, clientSecret string) string { + mac := hmac.New(sha256.New, []byte(clientSecret)) + mac.Write([]byte(timestamp)) + return base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} + +func buildDataToSign(req *http.Request, authBase string) string { + scheme := strings.ToLower(req.URL.Scheme) + if scheme == "" { + scheme = "https" + } + host := strings.ToLower(req.URL.Host) + path := req.URL.Path + if path == "" { + path = "/" + } + + var sb strings.Builder + sb.WriteString(req.Method) + sb.WriteString("\t") + sb.WriteString(scheme) + sb.WriteString("\t") + sb.WriteString(host) + sb.WriteString("\t") + sb.WriteString(path) + if req.URL.RawQuery != "" { + sb.WriteString("?") + sb.WriteString(req.URL.RawQuery) + } + sb.WriteString("\t\t\t") + sb.WriteString(authBase) + + return sb.String() +} + +func computeSignature(data, key string) string { + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte(data)) + return base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} + +// Transport wraps an http.RoundTripper to add EdgeGrid authentication. +type Transport struct { + Base http.RoundTripper + Signer *EdgeGridSigner +} + +// RoundTrip signs the request and delegates to the base transport. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + t.Signer.Sign(clone) + return t.Base.RoundTrip(clone) +} diff --git a/receiver/akamaisiemreceiver/internal/auth/edgegrid_test.go b/receiver/akamaisiemreceiver/internal/auth/edgegrid_test.go new file mode 100644 index 000000000..0ba74213b --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/auth/edgegrid_test.go @@ -0,0 +1,84 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEdgeGridSigner_Sign(t *testing.T) { + signer := NewEdgeGridSigner("client-token", "client-secret", "access-token") + + req, err := http.NewRequest(http.MethodGet, "https://test.luna.akamaiapis.net/siem/v1/configs/12345?limit=10000", nil) + require.NoError(t, err) + + signer.Sign(req) + + authHeader := req.Header.Get("Authorization") + assert.Contains(t, authHeader, "EG1-HMAC-SHA256") + assert.Contains(t, authHeader, "client_token=client-token") + assert.Contains(t, authHeader, "access_token=access-token") + assert.Contains(t, authHeader, "timestamp=") + assert.Contains(t, authHeader, "nonce=") + assert.Contains(t, authHeader, "signature=") +} + +func TestEdgeGridSigner_SignDeterministic(t *testing.T) { + signer := NewEdgeGridSigner("ct", "cs", "at") + + req1, _ := http.NewRequest(http.MethodGet, "https://test.example.com/path", nil) + req2, _ := http.NewRequest(http.MethodGet, "https://test.example.com/path", nil) + + signer.Sign(req1) + signer.Sign(req2) + + // Each signature should be unique (different nonce/timestamp). + assert.NotEqual(t, req1.Header.Get("Authorization"), req2.Header.Get("Authorization")) +} + +func TestTransport_RoundTrip(t *testing.T) { + signer := NewEdgeGridSigner("ct", "cs", "at") + + var capturedAuth string + transport := &Transport{ + Base: roundTripFunc(func(req *http.Request) (*http.Response, error) { + capturedAuth = req.Header.Get("Authorization") + return &http.Response{StatusCode: 200}, nil + }), + Signer: signer, + } + + req, _ := http.NewRequest(http.MethodGet, "https://test.example.com/path", nil) + resp, err := transport.RoundTrip(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + assert.Contains(t, capturedAuth, "EG1-HMAC-SHA256") + + // Original request should NOT have auth header (transport clones it). + assert.Empty(t, req.Header.Get("Authorization")) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} diff --git a/receiver/akamaisiemreceiver/internal/cursor/cursor.go b/receiver/akamaisiemreceiver/internal/cursor/cursor.go new file mode 100644 index 000000000..1dfcaeeaa --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/cursor/cursor.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package cursor persists the receiver's chain-based polling state across +// restarts via the OTel storage extension interface. +package cursor // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/cursor" + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "go.opentelemetry.io/collector/extension/xextension/storage" +) + +const cursorKey = "akamai_siem_cursor" + +// Cursor holds the chain-based state for resuming event collection. +type Cursor struct { + ChainFrom int64 `json:"chain_from,omitempty"` + ChainTo int64 `json:"chain_to,omitempty"` + CaughtUp bool `json:"caught_up,omitempty"` + LastOffset string `json:"last_offset,omitempty"` + OffsetObtainedAt time.Time `json:"offset_obtained_at,omitempty"` +} + +// IsOffsetStale returns true if the stored offset has exceeded the given TTL. +func (c *Cursor) IsOffsetStale(ttl time.Duration) bool { + if ttl == 0 || c.LastOffset == "" { + return false + } + return !c.OffsetObtainedAt.IsZero() && time.Since(c.OffsetObtainedAt) > ttl +} + +// ClearOffset resets offset fields for chain replay. +func (c *Cursor) ClearOffset() { + c.LastOffset = "" + c.OffsetObtainedAt = time.Time{} +} + +// CursorStore persists cursor state using the OTel storage extension interface. +type CursorStore struct { + client storage.Client +} + +// NewCursorStore creates a cursor store backed by the given storage client. +func NewCursorStore(client storage.Client) *CursorStore { + return &CursorStore{client: client} +} + +// Load retrieves the persisted cursor. Returns a zero-value cursor if none exists. +func (s *CursorStore) Load(ctx context.Context) (Cursor, error) { + data, err := s.client.Get(ctx, cursorKey) + if err != nil { + return Cursor{}, fmt.Errorf("failed to read cursor: %w", err) + } + if data == nil { + return Cursor{}, nil + } + var c Cursor + if err := json.Unmarshal(data, &c); err != nil { + return Cursor{}, fmt.Errorf("failed to unmarshal cursor: %w", err) + } + return c, nil +} + +// Save persists the cursor. +func (s *CursorStore) Save(ctx context.Context, c Cursor) error { + data, err := json.Marshal(c) + if err != nil { + return fmt.Errorf("failed to marshal cursor: %w", err) + } + return s.client.Set(ctx, cursorKey, data) +} + +// Close releases the storage client. +func (s *CursorStore) Close(ctx context.Context) error { + return s.client.Close(ctx) +} diff --git a/receiver/akamaisiemreceiver/internal/cursor/cursor_test.go b/receiver/akamaisiemreceiver/internal/cursor/cursor_test.go new file mode 100644 index 000000000..a98a62c3e --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/cursor/cursor_test.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/extension/xextension/storage" +) + +func TestCursor_IsOffsetStale(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + cursor Cursor + ttl time.Duration + want bool + }{ + { + name: "no offset", + cursor: Cursor{}, + ttl: 120 * time.Second, + want: false, + }, + { + name: "ttl disabled", + cursor: Cursor{LastOffset: "abc", OffsetObtainedAt: now.Add(-5 * time.Minute)}, + ttl: 0, + want: false, + }, + { + name: "fresh offset", + cursor: Cursor{LastOffset: "abc", OffsetObtainedAt: now.Add(-10 * time.Second)}, + ttl: 120 * time.Second, + want: false, + }, + { + name: "stale offset", + cursor: Cursor{LastOffset: "abc", OffsetObtainedAt: now.Add(-5 * time.Minute)}, + ttl: 120 * time.Second, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.cursor.IsOffsetStale(tt.ttl)) + }) + } +} + +func TestCursor_ClearOffset(t *testing.T) { + c := Cursor{ + LastOffset: "abc", + OffsetObtainedAt: time.Now(), + ChainFrom: 1000, + } + c.ClearOffset() + assert.Empty(t, c.LastOffset) + assert.True(t, c.OffsetObtainedAt.IsZero()) + assert.Equal(t, int64(1000), c.ChainFrom) // other fields untouched +} + +func TestCursorStore_SaveAndLoad(t *testing.T) { + store := NewCursorStore(newMemStorageClient()) + ctx := context.Background() + + // Load from empty store returns zero cursor. + c, err := store.Load(ctx) + require.NoError(t, err) + assert.Equal(t, Cursor{}, c) + + // Save and reload. + saved := Cursor{ + ChainFrom: 1000, + ChainTo: 2000, + CaughtUp: true, + LastOffset: "test-offset", + OffsetObtainedAt: time.Now().Truncate(time.Second), // JSON loses nanoseconds + } + require.NoError(t, store.Save(ctx, saved)) + + loaded, err := store.Load(ctx) + require.NoError(t, err) + assert.Equal(t, saved.ChainFrom, loaded.ChainFrom) + assert.Equal(t, saved.ChainTo, loaded.ChainTo) + assert.Equal(t, saved.CaughtUp, loaded.CaughtUp) + assert.Equal(t, saved.LastOffset, loaded.LastOffset) +} + +func TestCursorStore_Overwrite(t *testing.T) { + store := NewCursorStore(newMemStorageClient()) + ctx := context.Background() + + require.NoError(t, store.Save(ctx, Cursor{ChainFrom: 100})) + require.NoError(t, store.Save(ctx, Cursor{ChainFrom: 200})) + + loaded, err := store.Load(ctx) + require.NoError(t, err) + assert.Equal(t, int64(200), loaded.ChainFrom) +} + +func TestCursorStore_Close(t *testing.T) { + store := NewCursorStore(newMemStorageClient()) + assert.NoError(t, store.Close(context.Background())) +} + +func TestCursorStore_LoadCorruptData(t *testing.T) { + client := newMemStorageClient() + // Write invalid JSON directly to storage. + require.NoError(t, client.Set(context.Background(), cursorKey, []byte("{invalid"))) + + store := NewCursorStore(client) + _, err := store.Load(context.Background()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unmarshal") +} + +func TestCursorStore_NopClient(t *testing.T) { + // NopClient returns nil on Get — should return zero cursor. + store := NewCursorStore(storage.NewNopClient()) + ctx := context.Background() + + c, err := store.Load(ctx) + require.NoError(t, err) + assert.Equal(t, Cursor{}, c) + + // Save doesn't error (nop). + require.NoError(t, store.Save(ctx, Cursor{ChainFrom: 100})) +} + +// memStorageClient is a simple in-memory storage.Client for tests. +type memStorageClient struct { + data sync.Map +} + +func newMemStorageClient() *memStorageClient { + return &memStorageClient{} +} + +func (m *memStorageClient) Get(_ context.Context, key string) ([]byte, error) { + v, ok := m.data.Load(key) + if !ok { + return nil, nil + } + return v.([]byte), nil +} + +func (m *memStorageClient) Set(_ context.Context, key string, value []byte) error { + m.data.Store(key, value) + return nil +} + +func (m *memStorageClient) Delete(_ context.Context, key string) error { + m.data.Delete(key) + return nil +} + +func (m *memStorageClient) Batch(_ context.Context, ops ...*storage.Operation) error { + for _, op := range ops { + switch op.Type { + case storage.Get: + v, _ := m.data.Load(op.Key) + if v != nil { + op.Value = v.([]byte) + } + case storage.Set: + m.data.Store(op.Key, op.Value) + case storage.Delete: + m.data.Delete(op.Key) + } + } + return nil +} + +func (m *memStorageClient) Close(_ context.Context) error { + return nil +} diff --git a/receiver/akamaisiemreceiver/internal/metadata/generated_logs.go b/receiver/akamaisiemreceiver/internal/metadata/generated_logs.go new file mode 100644 index 000000000..3332753aa --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadata/generated_logs.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver" +) + +// LogsBuilder provides an interface for scrapers to report logs while taking care of all the transformations +// required to produce log representation defined in metadata and user config. +type LogsBuilder struct { + logsBuffer plog.Logs + logRecordsBuffer plog.LogRecordSlice + buildInfo component.BuildInfo // contains version information. +} + +// LogBuilderOption applies changes to default logs builder. +type LogBuilderOption interface { + apply(*LogsBuilder) +} + +func NewLogsBuilder(settings receiver.Settings) *LogsBuilder { + lb := &LogsBuilder{ + logsBuffer: plog.NewLogs(), + logRecordsBuffer: plog.NewLogRecordSlice(), + buildInfo: settings.BuildInfo, + } + + return lb +} + +// ResourceLogsOption applies changes to provided resource logs. +type ResourceLogsOption interface { + apply(plog.ResourceLogs) +} + +type resourceLogsOptionFunc func(plog.ResourceLogs) + +func (rlof resourceLogsOptionFunc) apply(rl plog.ResourceLogs) { + rlof(rl) +} + +// WithLogsResource sets the provided resource on the emitted ResourceLogs. +// It's recommended to use ResourceBuilder to create the resource. +func WithLogsResource(res pcommon.Resource) ResourceLogsOption { + return resourceLogsOptionFunc(func(rl plog.ResourceLogs) { + res.CopyTo(rl.Resource()) + }) +} + +// AppendLogRecord adds a log record to the logs builder. +func (lb *LogsBuilder) AppendLogRecord(lr plog.LogRecord) { + lr.MoveTo(lb.logRecordsBuffer.AppendEmpty()) +} + +// EmitForResource saves all the generated logs under a new resource and updates the internal state to be ready for +// recording another set of log records as part of another resource. This function can be helpful when one scraper +// needs to emit logs from several resources. Otherwise calling this function is not required, +// just `Emit` function can be called instead. +// Resource attributes should be provided as ResourceLogsOption arguments. +func (lb *LogsBuilder) EmitForResource(options ...ResourceLogsOption) { + rl := plog.NewResourceLogs() + ils := rl.ScopeLogs().AppendEmpty() + ils.Scope().SetName(ScopeName) + ils.Scope().SetVersion(lb.buildInfo.Version) + + for _, op := range options { + op.apply(rl) + } + + if lb.logRecordsBuffer.Len() > 0 { + lb.logRecordsBuffer.MoveAndAppendTo(ils.LogRecords()) + lb.logRecordsBuffer = plog.NewLogRecordSlice() + } + + if ils.LogRecords().Len() > 0 { + rl.MoveTo(lb.logsBuffer.ResourceLogs().AppendEmpty()) + } +} + +// Emit returns all the logs accumulated by the logs builder and updates the internal state to be ready for +// recording another set of logs. This function will be responsible for applying all the transformations required to +// produce logs representation defined in metadata and user config. +func (lb *LogsBuilder) Emit(options ...ResourceLogsOption) plog.Logs { + lb.EmitForResource(options...) + logs := lb.logsBuffer + lb.logsBuffer = plog.NewLogs() + return logs +} diff --git a/receiver/akamaisiemreceiver/internal/metadata/generated_logs_test.go b/receiver/akamaisiemreceiver/internal/metadata/generated_logs_test.go new file mode 100644 index 000000000..dfbf77a47 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadata/generated_logs_test.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +func TestLogsBuilderAppendLogRecord(t *testing.T) { + observedZapCore, _ := observer.New(zap.WarnLevel) + settings := receivertest.NewNopSettings(receivertest.NopType) + settings.Logger = zap.New(observedZapCore) + lb := NewLogsBuilder(settings) + + res := pcommon.NewResource() + + // append the first log record + lr := plog.NewLogRecord() + lr.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + lr.Attributes().PutStr("type", "log") + lr.Body().SetStr("the first log record") + + // append the second log record + lr2 := plog.NewLogRecord() + lr2.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + lr2.Attributes().PutStr("type", "event") + lr2.Body().SetStr("the second log record") + + lb.AppendLogRecord(lr) + lb.AppendLogRecord(lr2) + + logs := lb.Emit(WithLogsResource(res)) + assert.Equal(t, 1, logs.ResourceLogs().Len()) + + rl := logs.ResourceLogs().At(0) + assert.Equal(t, 1, rl.ScopeLogs().Len()) + + sl := rl.ScopeLogs().At(0) + assert.Equal(t, ScopeName, sl.Scope().Name()) + assert.Equal(t, lb.buildInfo.Version, sl.Scope().Version()) + + assert.Equal(t, 2, sl.LogRecords().Len()) + + attrVal, ok := sl.LogRecords().At(0).Attributes().Get("type") + assert.True(t, ok) + assert.Equal(t, "log", attrVal.Str()) + + assert.Equal(t, pcommon.ValueTypeStr, sl.LogRecords().At(0).Body().Type()) + assert.Equal(t, "the first log record", sl.LogRecords().At(0).Body().Str()) + + attrVal, ok = sl.LogRecords().At(1).Attributes().Get("type") + assert.True(t, ok) + assert.Equal(t, "event", attrVal.Str()) + + assert.Equal(t, pcommon.ValueTypeStr, sl.LogRecords().At(1).Body().Type()) + assert.Equal(t, "the second log record", sl.LogRecords().At(1).Body().Str()) +} diff --git a/receiver/akamaisiemreceiver/internal/metadata/generated_status.go b/receiver/akamaisiemreceiver/internal/metadata/generated_status.go new file mode 100644 index 000000000..13172e2b0 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadata/generated_status.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +// Package metadata contains the autogenerated telemetry and +// build information for the receiver/akamai_siem component. +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("akamai_siem") + ScopeName = "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver" +) + +const ( + LogsStability = component.StabilityLevelAlpha +) diff --git a/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry.go b/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry.go new file mode 100644 index 000000000..d495a6ba1 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry.go @@ -0,0 +1,195 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "errors" + "sync" + + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "go.opentelemetry.io/collector/component" +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter("github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver") +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer("github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver") +} + +// TelemetryBuilder provides an interface for components to report telemetry +// as defined in metadata and user config. +type TelemetryBuilder struct { + meter metric.Meter + mu sync.Mutex + registrations []metric.Registration + AkamaiSiemBytesReceived metric.Int64Counter + AkamaiSiemCursorPersists metric.Int64Counter + AkamaiSiemEventsEmitted metric.Int64Counter + AkamaiSiemEventsPerPage metric.Int64Histogram + AkamaiSiemEventsPerSecond metric.Float64Histogram + AkamaiSiemEventsReceived metric.Int64Counter + AkamaiSiemInvalidTimestampRetries metric.Int64Counter + AkamaiSiemOffsetExpired metric.Int64Counter + AkamaiSiemOffsetTTLDrops metric.Int64Counter + AkamaiSiemPageProcessingTime metric.Float64Histogram + AkamaiSiemPagesProcessed metric.Int64Counter + AkamaiSiemPollDuration metric.Float64Histogram + AkamaiSiemRecoveryAttempts metric.Int64Counter + AkamaiSiemRequestDuration metric.Float64Histogram + AkamaiSiemRequestErrors metric.Int64Counter + AkamaiSiemRequests metric.Int64Counter +} + +// TelemetryBuilderOption applies changes to default builder. +type TelemetryBuilderOption interface { + apply(*TelemetryBuilder) +} + +type telemetryBuilderOptionFunc func(mb *TelemetryBuilder) + +func (tbof telemetryBuilderOptionFunc) apply(mb *TelemetryBuilder) { + tbof(mb) +} + +// Shutdown unregister all registered callbacks for async instruments. +func (builder *TelemetryBuilder) Shutdown() { + builder.mu.Lock() + defer builder.mu.Unlock() + for _, reg := range builder.registrations { + reg.Unregister() + } +} + +// NewTelemetryBuilder provides a struct with methods to update all internal telemetry +// for a component +func NewTelemetryBuilder(settings component.TelemetrySettings, options ...TelemetryBuilderOption) (*TelemetryBuilder, error) { + builder := TelemetryBuilder{} + for _, op := range options { + op.apply(&builder) + } + builder.meter = Meter(settings) + var err, errs error + builder.AkamaiSiemBytesReceived, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.bytes_received", + metric.WithDescription("Total bytes received from the Akamai SIEM API [Alpha]"), + metric.WithUnit("By"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemCursorPersists, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.cursor_persists", + metric.WithDescription("Total successful cursor persist operations [Alpha]"), + metric.WithUnit("{persists}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemEventsEmitted, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.events_emitted", + metric.WithDescription("Total events successfully forwarded to the downstream consumer via ConsumeLogs [Alpha]"), + metric.WithUnit("{events}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemEventsPerPage, err = builder.meter.Int64Histogram( + "otelcol_akamai_siem.events_per_page", + metric.WithDescription("Number of events received per API response page [Alpha]"), + metric.WithUnit("{events}"), + metric.WithExplicitBucketBoundaries([]float64{0, 10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 600000}...), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemEventsPerSecond, err = builder.meter.Float64Histogram( + "otelcol_akamai_siem.events_per_second", + metric.WithDescription("Events per second throughput per poll cycle [Alpha]"), + metric.WithUnit("{events}/s"), + metric.WithExplicitBucketBoundaries([]float64{100, 500, 1000, 5000, 10000, 25000, 50000, 100000, 500000}...), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemEventsReceived, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.events_received", + metric.WithDescription("Total events received from the Akamai SIEM API [Alpha]"), + metric.WithUnit("{events}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemInvalidTimestampRetries, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.invalid_timestamp_retries", + metric.WithDescription("Total HMAC timestamp retries [Alpha]"), + metric.WithUnit("{retries}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemOffsetExpired, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.offset_expired", + metric.WithDescription("Total 416 offset out of range errors [Alpha]"), + metric.WithUnit("{errors}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemOffsetTTLDrops, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.offset_ttl_drops", + metric.WithDescription("Total proactive offset TTL expirations [Alpha]"), + metric.WithUnit("{drops}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemPageProcessingTime, err = builder.meter.Float64Histogram( + "otelcol_akamai_siem.page_processing_time", + metric.WithDescription("Time to process a page (NDJSON parse + body-map construction + ConsumeLogs) [Alpha]"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries([]float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 5, 10}...), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemPagesProcessed, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.pages_processed", + metric.WithDescription("Total API response pages processed [Alpha]"), + metric.WithUnit("{pages}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemPollDuration, err = builder.meter.Float64Histogram( + "otelcol_akamai_siem.poll_duration", + metric.WithDescription("Duration of a full poll iteration (may include multiple pages) [Alpha]"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries([]float64{0.1, 0.5, 1, 5, 10, 30, 60, 120, 300}...), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemRecoveryAttempts, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.recovery_attempts", + metric.WithDescription("Total recovery actions taken (416 replays, timestamp retries, from clamps) [Alpha]"), + metric.WithUnit("{attempts}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemRequestDuration, err = builder.meter.Float64Histogram( + "otelcol_akamai_siem.request_duration", + metric.WithDescription("Duration of API requests to the Akamai SIEM API [Alpha]"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries([]float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}...), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemRequestErrors, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.request_errors", + metric.WithDescription("Total failed API requests (non-200 responses) [Alpha]"), + metric.WithUnit("{requests}"), + ) + errs = errors.Join(errs, err) + builder.AkamaiSiemRequests, err = builder.meter.Int64Counter( + "otelcol_akamai_siem.requests", + metric.WithDescription("Total API requests made to the Akamai SIEM API [Alpha]"), + metric.WithUnit("{requests}"), + ) + errs = errors.Join(errs, err) + return &builder, errs +} diff --git a/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry_test.go b/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry_test.go new file mode 100644 index 000000000..14b57d543 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadata/generated_telemetry_test.go @@ -0,0 +1,91 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric" + embeddedmetric "go.opentelemetry.io/otel/metric/embedded" + noopmetric "go.opentelemetry.io/otel/metric/noop" + "go.opentelemetry.io/otel/trace" + embeddedtrace "go.opentelemetry.io/otel/trace/embedded" + nooptrace "go.opentelemetry.io/otel/trace/noop" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" +) + +type mockMeter struct { + noopmetric.Meter + name string +} +type mockMeterProvider struct { + embeddedmetric.MeterProvider +} + +func (m mockMeterProvider) Meter(name string, opts ...metric.MeterOption) metric.Meter { + return mockMeter{name: name} +} + +type mockTracer struct { + nooptrace.Tracer + name string +} + +type mockTracerProvider struct { + embeddedtrace.TracerProvider +} + +func (m mockTracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer { + return mockTracer{name: name} +} + +func TestProviders(t *testing.T) { + set := component.TelemetrySettings{ + MeterProvider: mockMeterProvider{}, + TracerProvider: mockTracerProvider{}, + } + + meter := Meter(set) + if m, ok := meter.(mockMeter); ok { + require.Equal(t, "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver", m.name) + } else { + require.Fail(t, "returned Meter not mockMeter") + } + + tracer := Tracer(set) + if m, ok := tracer.(mockTracer); ok { + require.Equal(t, "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver", m.name) + } else { + require.Fail(t, "returned Meter not mockTracer") + } +} + +func TestNewTelemetryBuilder(t *testing.T) { + set := componenttest.NewNopTelemetrySettings() + applied := false + _, err := NewTelemetryBuilder(set, telemetryBuilderOptionFunc(func(b *TelemetryBuilder) { + applied = true + })) + require.NoError(t, err) + require.True(t, applied) +} diff --git a/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest.go b/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest.go new file mode 100644 index 000000000..f1d670e58 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest.go @@ -0,0 +1,290 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadatatest + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" +) + +func NewSettings(tt *componenttest.Telemetry) receiver.Settings { + set := receivertest.NewNopSettings(receivertest.NopType) + set.ID = component.NewID(component.MustNewType("akamai_siem")) + set.TelemetrySettings = tt.NewTelemetrySettings() + return set +} + +func AssertEqualAkamaiSiemBytesReceived(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.bytes_received", + Description: "Total bytes received from the Akamai SIEM API [Alpha]", + Unit: "By", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.bytes_received") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemCursorPersists(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.cursor_persists", + Description: "Total successful cursor persist operations [Alpha]", + Unit: "{persists}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.cursor_persists") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemEventsEmitted(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.events_emitted", + Description: "Total events successfully forwarded to the downstream consumer via ConsumeLogs [Alpha]", + Unit: "{events}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.events_emitted") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemEventsPerPage(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.events_per_page", + Description: "Number of events received per API response page [Alpha]", + Unit: "{events}", + Data: metricdata.Histogram[int64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.events_per_page") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemEventsPerSecond(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[float64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.events_per_second", + Description: "Events per second throughput per poll cycle [Alpha]", + Unit: "{events}/s", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.events_per_second") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemEventsReceived(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.events_received", + Description: "Total events received from the Akamai SIEM API [Alpha]", + Unit: "{events}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.events_received") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemInvalidTimestampRetries(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.invalid_timestamp_retries", + Description: "Total HMAC timestamp retries [Alpha]", + Unit: "{retries}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.invalid_timestamp_retries") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemOffsetExpired(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.offset_expired", + Description: "Total 416 offset out of range errors [Alpha]", + Unit: "{errors}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.offset_expired") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemOffsetTTLDrops(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.offset_ttl_drops", + Description: "Total proactive offset TTL expirations [Alpha]", + Unit: "{drops}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.offset_ttl_drops") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemPageProcessingTime(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[float64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.page_processing_time", + Description: "Time to process a page (NDJSON parse + body-map construction + ConsumeLogs) [Alpha]", + Unit: "s", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.page_processing_time") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemPagesProcessed(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.pages_processed", + Description: "Total API response pages processed [Alpha]", + Unit: "{pages}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.pages_processed") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemPollDuration(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[float64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.poll_duration", + Description: "Duration of a full poll iteration (may include multiple pages) [Alpha]", + Unit: "s", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.poll_duration") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemRecoveryAttempts(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.recovery_attempts", + Description: "Total recovery actions taken (416 replays, timestamp retries, from clamps) [Alpha]", + Unit: "{attempts}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.recovery_attempts") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemRequestDuration(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.HistogramDataPoint[float64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.request_duration", + Description: "Duration of API requests to the Akamai SIEM API [Alpha]", + Unit: "s", + Data: metricdata.Histogram[float64]{ + Temporality: metricdata.CumulativeTemporality, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.request_duration") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemRequestErrors(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.request_errors", + Description: "Total failed API requests (non-200 responses) [Alpha]", + Unit: "{requests}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.request_errors") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} + +func AssertEqualAkamaiSiemRequests(t *testing.T, tt *componenttest.Telemetry, dps []metricdata.DataPoint[int64], opts ...metricdatatest.Option) { + want := metricdata.Metrics{ + Name: "otelcol_akamai_siem.requests", + Description: "Total API requests made to the Akamai SIEM API [Alpha]", + Unit: "{requests}", + Data: metricdata.Sum[int64]{ + Temporality: metricdata.CumulativeTemporality, + IsMonotonic: true, + DataPoints: dps, + }, + } + got, err := tt.GetMetric("otelcol_akamai_siem.requests") + require.NoError(t, err) + metricdatatest.AssertEqual(t, want, got, opts...) +} diff --git a/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest_test.go b/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest_test.go new file mode 100644 index 000000000..e50f3a6cd --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/metadatatest/generated_telemetrytest_test.go @@ -0,0 +1,105 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mdatagen. DO NOT EDIT. + +package metadatatest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" + + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/metadata" + "go.opentelemetry.io/collector/component/componenttest" +) + +func TestSetupTelemetry(t *testing.T) { + testTel := componenttest.NewTelemetry() + tb, err := metadata.NewTelemetryBuilder(testTel.NewTelemetrySettings()) + require.NoError(t, err) + defer tb.Shutdown() + tb.AkamaiSiemBytesReceived.Add(context.Background(), 1) + tb.AkamaiSiemCursorPersists.Add(context.Background(), 1) + tb.AkamaiSiemEventsEmitted.Add(context.Background(), 1) + tb.AkamaiSiemEventsPerPage.Record(context.Background(), 1) + tb.AkamaiSiemEventsPerSecond.Record(context.Background(), 1) + tb.AkamaiSiemEventsReceived.Add(context.Background(), 1) + tb.AkamaiSiemInvalidTimestampRetries.Add(context.Background(), 1) + tb.AkamaiSiemOffsetExpired.Add(context.Background(), 1) + tb.AkamaiSiemOffsetTTLDrops.Add(context.Background(), 1) + tb.AkamaiSiemPageProcessingTime.Record(context.Background(), 1) + tb.AkamaiSiemPagesProcessed.Add(context.Background(), 1) + tb.AkamaiSiemPollDuration.Record(context.Background(), 1) + tb.AkamaiSiemRecoveryAttempts.Add(context.Background(), 1) + tb.AkamaiSiemRequestDuration.Record(context.Background(), 1) + tb.AkamaiSiemRequestErrors.Add(context.Background(), 1) + tb.AkamaiSiemRequests.Add(context.Background(), 1) + AssertEqualAkamaiSiemBytesReceived(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemCursorPersists(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemEventsEmitted(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemEventsPerPage(t, testTel, + []metricdata.HistogramDataPoint[int64]{{}}, metricdatatest.IgnoreValue(), + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemEventsPerSecond(t, testTel, + []metricdata.HistogramDataPoint[float64]{{}}, metricdatatest.IgnoreValue(), + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemEventsReceived(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemInvalidTimestampRetries(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemOffsetExpired(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemOffsetTTLDrops(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemPageProcessingTime(t, testTel, + []metricdata.HistogramDataPoint[float64]{{}}, metricdatatest.IgnoreValue(), + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemPagesProcessed(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemPollDuration(t, testTel, + []metricdata.HistogramDataPoint[float64]{{}}, metricdatatest.IgnoreValue(), + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemRecoveryAttempts(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemRequestDuration(t, testTel, + []metricdata.HistogramDataPoint[float64]{{}}, metricdatatest.IgnoreValue(), + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemRequestErrors(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + AssertEqualAkamaiSiemRequests(t, testTel, + []metricdata.DataPoint[int64]{{Value: 1}}, + metricdatatest.IgnoreTimestamp()) + + require.NoError(t, testTel.Shutdown(context.Background())) +} diff --git a/receiver/akamaisiemreceiver/internal/poller/poller.go b/receiver/akamaisiemreceiver/internal/poller/poller.go new file mode 100644 index 000000000..c82eeb6ac --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/poller/poller.go @@ -0,0 +1,578 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package poller drives the Akamai SIEM API polling loop and implements the +// three-branch chain state machine (offset drain, chain replay, new chain). +package poller // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/poller" + +import ( + "context" + "errors" + "io" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" + + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/akamaiclient" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/cursor" +) + +// Telemetry provides optional metric and tracing instruments for the poller. +// All fields are optional — nil values are silently skipped. +type Telemetry struct { + Tracer trace.Tracer + Requests metric.Int64Counter + RequestErrors metric.Int64Counter + EventsReceived metric.Int64Counter + EventsEmitted metric.Int64Counter + OffsetExpired metric.Int64Counter + OffsetTTLDrops metric.Int64Counter + RecoveryAttempts metric.Int64Counter + InvalidTSRetries metric.Int64Counter + RequestDuration metric.Float64Histogram + PollDuration metric.Float64Histogram + EventsPerSecond metric.Float64Histogram + PagesProcessed metric.Int64Counter + CursorPersists metric.Int64Counter + BytesReceived metric.Int64Counter + PageProcessingTime metric.Float64Histogram + EventsPerPage metric.Int64Histogram +} + +func (t *Telemetry) addCounter(ctx context.Context, c metric.Int64Counter, v int64) { + if t != nil && c != nil { + c.Add(ctx, v) + } +} + +func (t *Telemetry) recordFloat(ctx context.Context, h metric.Float64Histogram, v float64) { + if t != nil && h != nil { + h.Record(ctx, v) + } +} + +func (t *Telemetry) recordInt(ctx context.Context, h metric.Int64Histogram, v int64) { + if t != nil && h != nil { + h.Record(ctx, v) + } +} + +// startSpan starts a trace span if a tracer is configured. Returns ctx and a +// nil-safe end function. When no tracer is set, this is a no-op. +func (t *Telemetry) startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, func(error)) { + if t == nil || t.Tracer == nil { + return ctx, func(error) {} + } + ctx, span := t.Tracer.Start(ctx, name, trace.WithAttributes(attrs...)) + return ctx, func(err error) { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + } +} + +// Time-window constants are stored as Unix-second counts so buildFetchParams +// does not derive them from time.Duration on every call. +const ( + chainOverlapSec int64 = 10 // 10 seconds — overlap when re-anchoring a chain + maxLookbackSec int64 = 12 * 60 * 60 // 12 hours — Akamai's max accepted lookback + + // apiSafetyBuffer is subtracted from "now" when computing the `to` parameter + // for new chain windows. Akamai's SIEM API has a multi-second write-visibility + // delay: events take time to be indexed before they appear in API responses. + // Querying to=now would return an incomplete tail and silently skip events on + // the next poll, because `from` advances past them. 60s is conservative based + // on observed Akamai latency; reducing it risks dropped events. + apiSafetyBuffer int64 = 60 +) + +// maxSpanBodyBytes caps the response body length recorded as a span attribute. +// Proxy/HTML error pages can be tens of KB; spans aren't a place for that. +const maxSpanBodyBytes = 2048 + +// annotateFetchSpan attaches APIError details (status_code, detail, truncated +// raw body) to the FetchPage span when the fetch error is an *APIError. For +// non-APIError errors, the span's recorded error event already carries the +// message via err.Error() — nothing to add. +func annotateFetchSpan(ctx context.Context, err error) { + var apiErr *akamaiclient.APIError + if !errors.As(err, &apiErr) { + return + } + span := trace.SpanFromContext(ctx) + attrs := []attribute.KeyValue{ + attribute.Int("akamai.api.status_code", apiErr.StatusCode), + } + if apiErr.Detail != "" { + attrs = append(attrs, attribute.String("akamai.api.detail", apiErr.Detail)) + } + if apiErr.Body != "" { + body := apiErr.Body + truncated := false + if len(body) > maxSpanBodyBytes { + body = body[:maxSpanBodyBytes] + truncated = true + } + attrs = append(attrs, attribute.String("akamai.api.body", body)) + if truncated { + attrs = append(attrs, attribute.Bool("akamai.api.body_truncated", true)) + } + } + span.SetAttributes(attrs...) +} + +// clampToMaxLookback returns from clamped to now - maxLookbackSec, the earliest +// timestamp Akamai will accept. Both arguments are Unix-seconds. +func clampToMaxLookback(from, now int64) int64 { + earliest := now - maxLookbackSec + if from < earliest { + return earliest + } + return from +} + +// PollerConfig holds the parameters that drive the polling state machine. +type PollerConfig struct { + EventLimit int + InitialLookback time.Duration + OffsetTTL time.Duration + MaxRecoveryAttempts int + InvalidTimestampRetries int + BatchSize int // events per ConsumeLogs call (default 1000) + StreamBufferSize int // bounded channel capacity (default 4) +} + +// EventEmitter is called by the poller to emit a batch of raw JSON events. +// The implementation is responsible for converting them to plog.Logs and +// calling ConsumeLogs. +type EventEmitter func(ctx context.Context, events []string) error + +// Poller implements the three-branch chain state machine for the Akamai SIEM API. +type Poller struct { + client *akamaiclient.Client + cursor cursor.Cursor + cursorStore *cursor.CursorStore + cfg PollerConfig + emit EventEmitter + log *zap.Logger + telemetry *Telemetry +} + +// NewPoller creates a new poller. +func NewPoller(client *akamaiclient.Client, cursorStore *cursor.CursorStore, cur cursor.Cursor, cfg PollerConfig, emit EventEmitter, log *zap.Logger, telemetry *Telemetry) *Poller { + return &Poller{ + client: client, + cursor: cur, + cursorStore: cursorStore, + cfg: cfg, + emit: emit, + log: log.Named("poller"), + telemetry: telemetry, + } +} + +// Poll performs a single polling iteration, fetching pages until the chain +// is drained (events < event_limit). +func (p *Poller) Poll(ctx context.Context) error { + ctx, endSpan := p.telemetry.startSpan(ctx, "akamai_siem.Poll", + attribute.Bool("cursor.caught_up", p.cursor.CaughtUp), + attribute.String("cursor.last_offset", p.cursor.LastOffset), + ) + start := time.Now() + p.log.Debug("starting poll iteration", + zap.Int64("chain_from", p.cursor.ChainFrom), + zap.Int64("chain_to", p.cursor.ChainTo), + zap.Bool("caught_up", p.cursor.CaughtUp), + zap.String("last_offset", p.cursor.LastOffset), + ) + + params := p.buildFetchParams(ctx) + pageCount := 0 + pollEventsTotal := 0 + recoveryAttempts := 0 + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + fetchCtx, endFetchSpan := p.telemetry.startSpan(ctx, "akamai_siem.FetchPage", + attribute.String("mode", params.FetchMode()), + attribute.Int("limit", params.Limit), + ) + body, fetchErr := p.fetchWithTimestampRetry(fetchCtx, params) + if fetchErr != nil { + annotateFetchSpan(fetchCtx, fetchErr) + endFetchSpan(fetchErr) + p.telemetry.addCounter(ctx, p.telemetry.RequestErrors, 1) + if !p.handleFetchError(ctx, fetchErr, ¶ms) { + return nil + } + p.telemetry.addCounter(ctx, p.telemetry.RecoveryAttempts, 1) + recoveryAttempts++ + if p.cfg.MaxRecoveryAttempts > 0 && recoveryAttempts >= p.cfg.MaxRecoveryAttempts { + p.log.Error("max recovery attempts reached, ending poll cycle", + zap.Int("recovery_attempts", recoveryAttempts), + zap.Error(fetchErr), + ) + return nil + } + continue + } + endFetchSpan(nil) + recoveryAttempts = 0 + pageCount++ + + eventCount, pageCtx, processErr := p.processPage(ctx, body, pageCount, params) + _ = body.Close() + if processErr != nil { + return nil + } + pollEventsTotal += eventCount + + if eventCount == 0 { + p.cursor.CaughtUp = true + p.log.Debug("no events received, chain drained") + p.persistCursor(ctx) + break + } + + // Update cursor with page offset. + if pageCtx.Offset != "" { + p.cursor.LastOffset = pageCtx.Offset + p.cursor.OffsetObtainedAt = time.Now() + } + + // Drain detection. + p.cursor.CaughtUp = eventCount < p.cfg.EventLimit + p.persistCursor(ctx) + + if p.cursor.CaughtUp { + p.log.Debug("chain drained", + zap.Int("events", eventCount), + zap.Int("limit", p.cfg.EventLimit), + ) + break + } + + if pageCtx.Offset == "" { + p.log.Error("missing next offset in paginated response; ending cycle") + break + } + + // Continue draining with next page. From/To reset is redundant with + // buildRequestURL (which only emits from/to when Offset is empty), but + // we zero them here so the FetchParams value reads cleanly in logs. + params.Offset = pageCtx.Offset + params.From = 0 + params.To = 0 + } + + elapsed := time.Since(start) + p.telemetry.recordFloat(ctx, p.telemetry.PollDuration, elapsed.Seconds()) + + // EPS: events per second for this poll cycle. + if elapsed.Seconds() > 0 && pollEventsTotal > 0 { + eps := float64(pollEventsTotal) / elapsed.Seconds() + p.telemetry.recordFloat(ctx, p.telemetry.EventsPerSecond, eps) + } + + p.log.Debug("poll iteration complete", + zap.Duration("duration", elapsed), + zap.Int("pages", pageCount), + zap.Int("events", pollEventsTotal), + zap.Bool("caught_up", p.cursor.CaughtUp), + ) + endSpan(nil) + return nil +} + +func (p *Poller) persistCursor(ctx context.Context) { + if p.cursorStore == nil { + return + } + _, endSpan := p.telemetry.startSpan(ctx, "akamai_siem.PersistCursor", + attribute.String("last_offset", p.cursor.LastOffset), + attribute.Bool("caught_up", p.cursor.CaughtUp), + ) + err := p.cursorStore.Save(ctx, p.cursor) + endSpan(err) + if err != nil { + p.log.Error("failed to persist cursor", zap.Error(err)) + } else { + p.telemetry.addCounter(ctx, p.telemetry.CursorPersists, 1) + p.log.Debug("cursor persisted", + zap.Int64("chain_from", p.cursor.ChainFrom), + zap.Int64("chain_to", p.cursor.ChainTo), + zap.String("last_offset", p.cursor.LastOffset), + zap.Bool("caught_up", p.cursor.CaughtUp), + ) + } +} + +// processPage streams events from body through a bounded channel. A consumer +// goroutine reads from the channel, batches events, and calls the EventEmitter +// per batch. This bounds memory to (streamBufferSize + batchSize) events +// regardless of total page size. +// +// Cursor is NOT persisted here — the caller handles that after processPage +// returns, so cursor persist only happens after all events in the page +// are confirmed. +func (p *Poller) processPage(ctx context.Context, body interface{ Read([]byte) (int, error) }, page int, params akamaiclient.FetchParams) (int, akamaiclient.OffsetContext, error) { + _, endSpan := p.telemetry.startSpan(ctx, "akamai_siem.ProcessPage", + attribute.Int("page", page), + ) + pageStart := time.Now() + + // StreamBufferSize and BatchSize are validated > 0 in Config.Validate. + eventCh := make(chan string, p.cfg.StreamBufferSize) + batchSize := p.cfg.BatchSize + + // Scanner goroutine: streams NDJSON lines into bounded channel. + // Closes eventCh when done so the consumer's range loop exits. + var pageCtx akamaiclient.OffsetContext + var streamCount int + var streamErr error + go func() { + defer close(eventCh) + pageCtx, streamCount, streamErr = akamaiclient.StreamEvents(ctx, body, eventCh) + }() + + // Consumer: reads from channel, batches, calls emit per batch. + // Runs on the calling goroutine — range exits when scanner closes eventCh. + // emittedCount is goroutine-local; the scanner goroutine never touches it. + var emitErr error + emittedCount := 0 + batch := make([]string, 0, batchSize) + + for event := range eventCh { + batch = append(batch, event) + if len(batch) >= batchSize { + if err := p.emit(ctx, batch); err != nil { + emitErr = err + for range eventCh { + } + break + } + emittedCount += len(batch) + batch = make([]string, 0, batchSize) + } + } + if emitErr == nil && len(batch) > 0 { + if err := p.emit(ctx, batch); err != nil { + emitErr = err + } else { + emittedCount += len(batch) + } + } + + totalEmitted := emittedCount + + // Handle errors. + if streamErr != nil { + endSpan(streamErr) + p.log.Error("failed to stream events", + zap.Error(streamErr), + zap.Int("page", page), + zap.String("mode", params.FetchMode()), + ) + return 0, akamaiclient.OffsetContext{}, streamErr + } + if emitErr != nil { + endSpan(emitErr) + p.log.Error("failed to emit events", + zap.Error(emitErr), + zap.Int("events_emitted", totalEmitted), + zap.Int("events_received", streamCount), + zap.Int("page", page), + ) + return 0, akamaiclient.OffsetContext{}, emitErr + } + + // Telemetry. + p.telemetry.addCounter(ctx, p.telemetry.EventsReceived, int64(streamCount)) + p.telemetry.addCounter(ctx, p.telemetry.PagesProcessed, 1) + if cl := p.client.LastContentLength(); cl > 0 { + p.telemetry.addCounter(ctx, p.telemetry.BytesReceived, cl) + } + p.telemetry.addCounter(ctx, p.telemetry.EventsEmitted, int64(totalEmitted)) + p.telemetry.recordInt(ctx, p.telemetry.EventsPerPage, int64(streamCount)) + pageElapsed := time.Since(pageStart) + p.telemetry.recordFloat(ctx, p.telemetry.PageProcessingTime, pageElapsed.Seconds()) + endSpan(nil) + + p.log.Debug("page processed", + zap.Int("page", page), + zap.Int("events_received", streamCount), + zap.Int("events_emitted", totalEmitted), + zap.Duration("processing_time", pageElapsed), + zap.String("offset", pageCtx.Offset), + ) + + return streamCount, pageCtx, nil +} + +// buildFetchParams implements the three-branch chain state machine. +func (p *Poller) buildFetchParams(ctx context.Context) akamaiclient.FetchParams { + now := time.Now().Unix() + params := akamaiclient.FetchParams{Limit: p.cfg.EventLimit} + + switch { + case !p.cursor.CaughtUp && p.cursor.LastOffset != "" && !p.cursor.IsOffsetStale(p.cfg.OffsetTTL): + // Branch 1: Chain in progress, offset valid — continue draining. + params.Offset = p.cursor.LastOffset + p.log.Debug("offset-based fetch (chain draining)", + zap.String("offset", params.Offset), + ) + + case !p.cursor.CaughtUp && p.cursor.ChainFrom != 0: + // Branch 2: Chain in progress but offset gone/stale — replay chain window. + // Logs a warning when the clamp activates; chain-replay clamping signals + // that an in-flight chain has aged past the API window and we may be + // dropping events. + if p.cursor.IsOffsetStale(p.cfg.OffsetTTL) { + p.log.Warn("offset stale, replaying chain window", + zap.Duration("offset_age", time.Since(p.cursor.OffsetObtainedAt)), + zap.Duration("ttl", p.cfg.OffsetTTL), + ) + p.telemetry.addCounter(ctx, p.telemetry.OffsetTTLDrops, 1) + } + p.cursor.ClearOffset() + + from := clampToMaxLookback(p.cursor.ChainFrom-chainOverlapSec, now) + if from != p.cursor.ChainFrom-chainOverlapSec { + p.log.Warn("chain_from clamped to max lookback", + zap.Int64("original_from", p.cursor.ChainFrom), + zap.Int64("clamped_from", from), + ) + } + params.From = from + params.To = p.cursor.ChainTo + p.log.Debug("time-based fetch (chain replay)", + zap.Int64("from", params.From), + zap.Int64("to", params.To), + ) + + default: + // Branch 3: Caught up or first run — start a new chain. Clamping is + // expected here on first run with a long initial_lookback; no warning. + var from int64 + if p.cursor.ChainTo != 0 { + from = p.cursor.ChainTo - chainOverlapSec + } else { + from = now - int64(p.cfg.InitialLookback.Seconds()) + } + from = clampToMaxLookback(from, now) + to := now - apiSafetyBuffer + + p.cursor.ChainFrom = from + p.cursor.ChainTo = to + p.cursor.CaughtUp = false + p.cursor.ClearOffset() + + params.From = from + params.To = to + p.log.Debug("time-based fetch (new chain)", + zap.Int64("from", params.From), + zap.Int64("to", params.To), + ) + } + + return params +} + +func (p *Poller) fetchWithTimestampRetry(ctx context.Context, params akamaiclient.FetchParams) (io.ReadCloser, error) { + maxRetries := p.cfg.InvalidTimestampRetries + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + p.telemetry.addCounter(ctx, p.telemetry.InvalidTSRetries, 1) + p.log.Debug("retrying request after invalid timestamp", + zap.Int("attempt", attempt), + ) + } + + p.telemetry.addCounter(ctx, p.telemetry.Requests, 1) + reqStart := time.Now() + body, err := p.client.FetchResponse(ctx, params) + p.telemetry.recordFloat(ctx, p.telemetry.RequestDuration, time.Since(reqStart).Seconds()) + if err == nil { + return body, nil + } + lastErr = err + + var apiErr *akamaiclient.APIError + if errors.As(err, &apiErr) && apiErr.IsInvalidTimestamp() && attempt < maxRetries { + continue + } + return nil, err + } + return nil, lastErr +} + +// handleFetchError processes API errors. Returns true if recoverable. +func (p *Poller) handleFetchError(ctx context.Context, err error, params *akamaiclient.FetchParams) bool { + var apiErr *akamaiclient.APIError + if !errors.As(err, &apiErr) { + p.log.Error("failed to fetch events", zap.Error(err)) + return false + } + + switch { + case apiErr.IsOffsetOutOfRange(): + p.log.Warn("416 offset expired; clearing offset for chain replay", + zap.String("last_offset", p.cursor.LastOffset), + ) + p.telemetry.addCounter(ctx, p.telemetry.OffsetExpired, 1) + p.cursor.ClearOffset() + *params = p.buildFetchParams(ctx) + return true + + case apiErr.IsInvalidTimestamp(): + p.log.Warn("invalid timestamp after retries; clearing offset for chain replay") + p.cursor.ClearOffset() + *params = p.buildFetchParams(ctx) + return true + + case apiErr.IsFromTooOld(): + p.log.Warn("from timestamp too old, replaying with clamp") + *params = p.buildFetchParams(ctx) + return true + + case apiErr.StatusCode == 400: + p.log.Error("non-recoverable 400 response", + zap.Int("status_code", apiErr.StatusCode), + zap.String("detail", apiErr.Detail), + ) + return false + + default: + p.log.Error("failed to fetch events", + zap.Int("status_code", apiErr.StatusCode), + zap.String("detail", apiErr.Detail), + ) + return false + } +} diff --git a/receiver/akamaisiemreceiver/internal/poller/poller_test.go b/receiver/akamaisiemreceiver/internal/poller/poller_test.go new file mode 100644 index 000000000..e1c5bc5a6 --- /dev/null +++ b/receiver/akamaisiemreceiver/internal/poller/poller_test.go @@ -0,0 +1,548 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package poller + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/extension/xextension/storage" + "go.opentelemetry.io/otel/metric/noop" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.uber.org/zap/zaptest" + + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/akamaiclient" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/cursor" +) + +func TestPoller_Poll_BasicFlow(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"cursor-1","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + store := cursor.NewCursorStore(newMemStorageClient()) + + var emitted []string + emit := func(_ context.Context, events []string) error { + emitted = append(emitted, events...) + return nil + } + + poller := NewPoller(client, store, cursor.Cursor{}, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + BatchSize: 1000, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) + assert.Len(t, emitted, 1) + assert.Contains(t, emitted[0], `"host":"test.com"`) + + // Cursor should be persisted. + cursor, err := store.Load(context.Background()) + require.NoError(t, err) + assert.Equal(t, "cursor-1", cursor.LastOffset) + assert.True(t, cursor.CaughtUp) +} + +func TestPoller_Poll_MultiPageDrain(t *testing.T) { + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := int(requestCount.Add(1)) + switch page { + case 1: + _, _ = fmt.Fprintln(w, `{"event":"p1e1"}`) + _, _ = fmt.Fprintln(w, `{"event":"p1e2"}`) + _, _ = fmt.Fprintln(w, `{"offset":"page2","total":2,"limit":2}`) + case 2: + _, _ = fmt.Fprintln(w, `{"event":"p2e1"}`) + _, _ = fmt.Fprintln(w, `{"offset":"done","total":1,"limit":2}`) + default: + w.WriteHeader(200) + } + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + var emitted []string + emit := func(_ context.Context, events []string) error { + emitted = append(emitted, events...) + return nil + } + + poller := NewPoller(client, nil, cursor.Cursor{}, PollerConfig{ + EventLimit: 2, + InitialLookback: 1 * time.Hour, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) + assert.Len(t, emitted, 3) + assert.GreaterOrEqual(t, int(requestCount.Load()), 2) +} + +func TestPoller_Poll_EmitError_NoCursorPersist(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"event":"data"}`) + _, _ = fmt.Fprintln(w, `{"offset":"x","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + store := cursor.NewCursorStore(newMemStorageClient()) + + emit := func(_ context.Context, _ []string) error { + return fmt.Errorf("emit failed") + } + + poller := NewPoller(client, store, cursor.Cursor{}, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) // Poll returns nil on emit error (logs it) + + // Cursor should NOT be persisted. + cursor, err := store.Load(context.Background()) + require.NoError(t, err) + assert.Empty(t, cursor.LastOffset) +} + +func TestPoller_Poll_416Recovery(t *testing.T) { + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := int(requestCount.Add(1)) + if count == 1 { + w.WriteHeader(416) + _, _ = fmt.Fprintln(w, `{"detail":"offset out of range"}`) + return + } + _, _ = fmt.Fprintln(w, `{"event":"recovered"}`) + _, _ = fmt.Fprintln(w, `{"offset":"new","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + var emitted []string + emit := func(_ context.Context, events []string) error { + emitted = append(emitted, events...) + return nil + } + + // Start with a stale offset to trigger 416. + cur := cursor.Cursor{ + ChainFrom: time.Now().Add(-1 * time.Hour).Unix(), + ChainTo: time.Now().Add(-1 * time.Minute).Unix(), + CaughtUp: false, + LastOffset: "stale-offset", + } + + poller := NewPoller(client, nil, cur, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + MaxRecoveryAttempts: 3, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) + assert.Len(t, emitted, 1) + assert.Contains(t, emitted[0], `"recovered"`) +} + +func TestPoller_Poll_MaxRecoveryAttempts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(416) + _, _ = fmt.Fprintln(w, `{"detail":"always expired"}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + emit := func(_ context.Context, _ []string) error { return nil } + + cur := cursor.Cursor{ + ChainFrom: time.Now().Add(-1 * time.Hour).Unix(), + ChainTo: time.Now().Add(-1 * time.Minute).Unix(), + CaughtUp: false, + LastOffset: "stale", + } + + poller := NewPoller(client, nil, cur, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + MaxRecoveryAttempts: 2, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) // Returns nil after max attempts +} + +func TestPoller_Poll_ContextCanceled(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"event":"data"}`) + _, _ = fmt.Fprintln(w, `{"offset":"x","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + emit := func(_ context.Context, _ []string) error { return nil } + + poller := NewPoller(client, nil, cursor.Cursor{}, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + err = poller.Poll(ctx) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestPoller_BuildFetchParams_Branch1_OffsetDrain(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ + CaughtUp: false, + LastOffset: "my-offset", + OffsetObtainedAt: time.Now(), + ChainFrom: 1000, + ChainTo: 2000, + }, + cfg: PollerConfig{ + EventLimit: 10000, + OffsetTTL: 120 * time.Second, + }, + log: log, + telemetry: nil, + } + + params := poller.buildFetchParams(context.Background()) + assert.Equal(t, "my-offset", params.Offset) + assert.Zero(t, params.From) + assert.Zero(t, params.To) +} + +func TestPoller_BuildFetchParams_Branch2_StaleOffset(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ + CaughtUp: false, + LastOffset: "stale-offset", + OffsetObtainedAt: time.Now().Add(-5 * time.Minute), + ChainFrom: time.Now().Add(-30 * time.Minute).Unix(), + ChainTo: time.Now().Add(-1 * time.Minute).Unix(), + }, + cfg: PollerConfig{ + EventLimit: 10000, + OffsetTTL: 30 * time.Second, // TTL expired + }, + log: log, + telemetry: &Telemetry{}, + } + + params := poller.buildFetchParams(context.Background()) + assert.Empty(t, params.Offset) // Offset cleared due to TTL + assert.Greater(t, params.From, int64(0)) + assert.Empty(t, poller.cursor.LastOffset) +} + +func TestPoller_BuildFetchParams_Branch3_ResumeFromChainTo(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ + CaughtUp: true, + ChainTo: time.Now().Add(-5 * time.Minute).Unix(), + }, + cfg: PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + }, + log: log, + telemetry: &Telemetry{}, + } + + params := poller.buildFetchParams(context.Background()) + // Should start from ChainTo - overlap, not InitialLookback + assert.Greater(t, params.From, poller.cursor.ChainTo-int64((12*time.Hour).Seconds())) +} + +func TestPoller_BuildFetchParams_Branch2_ChainReplay(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ + CaughtUp: false, + LastOffset: "", + ChainFrom: time.Now().Add(-30 * time.Minute).Unix(), + ChainTo: time.Now().Add(-1 * time.Minute).Unix(), + }, + cfg: PollerConfig{ + EventLimit: 10000, + InitialLookback: 12 * time.Hour, + }, + log: log, + telemetry: nil, + } + + params := poller.buildFetchParams(context.Background()) + assert.Empty(t, params.Offset) + assert.Greater(t, params.From, int64(0)) + assert.Greater(t, params.To, int64(0)) +} + +func TestPoller_BuildFetchParams_Branch3_NewChain(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ + CaughtUp: true, + }, + cfg: PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + }, + log: log, + telemetry: nil, + } + + params := poller.buildFetchParams(context.Background()) + assert.Empty(t, params.Offset) + assert.Greater(t, params.From, int64(0)) + assert.Greater(t, params.To, int64(0)) + assert.False(t, poller.cursor.CaughtUp) +} + +func TestTelemetry_WithRealInstruments(t *testing.T) { + meter := noop.NewMeterProvider().Meter("test") + counter, _ := meter.Int64Counter("test_counter") + histogram, _ := meter.Float64Histogram("test_histogram") + intHist, _ := meter.Int64Histogram("test_int_hist") + + tp := sdktrace.NewTracerProvider() + defer func() { _ = tp.Shutdown(context.Background()) }() + tracer := tp.Tracer("test") + + tel := &Telemetry{ + Tracer: tracer, + Requests: counter, + RequestErrors: counter, + EventsReceived: counter, + EventsEmitted: counter, + RequestDuration: histogram, + PollDuration: histogram, + EventsPerPage: intHist, + } + + ctx := context.Background() + + // Exercise all telemetry helpers with real instruments. + tel.addCounter(ctx, tel.Requests, 1) + tel.addCounter(ctx, tel.RequestErrors, 5) + tel.recordFloat(ctx, tel.RequestDuration, 0.5) + tel.recordInt(ctx, tel.EventsPerPage, 1000) + + // Span with real tracer. + spanCtx, endSpan := tel.startSpan(ctx, "test.span") + assert.NotEqual(t, ctx, spanCtx) // Should have a new span context + endSpan(nil) + + // Span with error. + _, endErrSpan := tel.startSpan(ctx, "test.error_span") + endErrSpan(fmt.Errorf("test error")) + + // Nil telemetry should not panic. + var nilTel *Telemetry + nilTel.addCounter(ctx, nil, 1) + nilTel.recordFloat(ctx, nil, 1.0) + nilTel.recordInt(ctx, nil, 1) + nilCtx, nilEnd := nilTel.startSpan(ctx, "noop") + assert.Equal(t, ctx, nilCtx) // No-op returns same context + nilEnd(nil) +} + +func TestPoller_HandleFetchError_NonAPI(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{log: log, telemetry: nil} + params := akamaiclient.FetchParams{Limit: 100} + result := poller.handleFetchError(context.Background(), fmt.Errorf("network error"), ¶ms) + assert.False(t, result) // Non-API errors are not recoverable +} + +func TestPoller_HandleFetchError_400Fatal(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{log: log, telemetry: nil} + params := akamaiclient.FetchParams{Limit: 100} + err := &akamaiclient.APIError{StatusCode: 400, Detail: "bad request"} + result := poller.handleFetchError(context.Background(), err, ¶ms) + assert.False(t, result) +} + +func TestPoller_Poll_InvalidTimestampRetry(t *testing.T) { + var requestCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := int(requestCount.Add(1)) + if count <= 2 { + w.WriteHeader(400) + _, _ = fmt.Fprintln(w, `{"detail":"invalid timestamp"}`) + return + } + _, _ = fmt.Fprintln(w, `{"event":"after-retry"}`) + _, _ = fmt.Fprintln(w, `{"offset":"ok","total":1,"limit":10000}`) + })) + defer server.Close() + + log := zaptest.NewLogger(t) + client, err := akamaiclient.NewClient(&http.Client{}, server.URL, "1", log) + require.NoError(t, err) + + var emitted []string + emit := func(_ context.Context, events []string) error { + emitted = append(emitted, events...) + return nil + } + + poller := NewPoller(client, nil, cursor.Cursor{}, PollerConfig{ + EventLimit: 10000, + InitialLookback: 1 * time.Hour, + InvalidTimestampRetries: 3, + MaxRecoveryAttempts: 5, + BatchSize: 100, + StreamBufferSize: 4, + }, emit, log, &Telemetry{}) + + err = poller.Poll(context.Background()) + require.NoError(t, err) + assert.Len(t, emitted, 1) + assert.GreaterOrEqual(t, int(requestCount.Load()), 3) +} + +func TestPoller_HandleFetchError_InvalidTimestamp(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ChainFrom: 1000, ChainTo: 2000, LastOffset: "old"}, + cfg: PollerConfig{EventLimit: 10000, InitialLookback: 1 * time.Hour}, + log: log, + telemetry: &Telemetry{}, + } + params := akamaiclient.FetchParams{Limit: 100} + err := &akamaiclient.APIError{StatusCode: 400, Detail: "invalid timestamp"} + result := poller.handleFetchError(context.Background(), err, ¶ms) + assert.True(t, result) + assert.Empty(t, poller.cursor.LastOffset) // Offset cleared +} + +func TestPoller_HandleFetchError_FromTooOld(t *testing.T) { + log := zaptest.NewLogger(t) + poller := &Poller{ + cursor: cursor.Cursor{ChainFrom: 1000, ChainTo: 2000}, + cfg: PollerConfig{EventLimit: 10000, InitialLookback: 1 * time.Hour}, + log: log, + telemetry: nil, + } + params := akamaiclient.FetchParams{Limit: 100} + err := &akamaiclient.APIError{StatusCode: 400, Detail: "from parameter is out of range"} + result := poller.handleFetchError(context.Background(), err, ¶ms) + assert.True(t, result) // Recoverable +} + +// memStorageClient is a simple in-memory storage.Client for tests. +type memStorageClient struct { + data sync.Map +} + +func newMemStorageClient() *memStorageClient { + return &memStorageClient{} +} + +func (m *memStorageClient) Get(_ context.Context, key string) ([]byte, error) { + v, ok := m.data.Load(key) + if !ok { + return nil, nil + } + return v.([]byte), nil +} + +func (m *memStorageClient) Set(_ context.Context, key string, value []byte) error { + m.data.Store(key, value) + return nil +} + +func (m *memStorageClient) Delete(_ context.Context, key string) error { + m.data.Delete(key) + return nil +} + +func (m *memStorageClient) Batch(_ context.Context, ops ...*storage.Operation) error { + for _, op := range ops { + switch op.Type { + case storage.Get: + v, _ := m.data.Load(op.Key) + if v != nil { + op.Value = v.([]byte) + } + case storage.Set: + m.data.Store(op.Key, op.Value) + case storage.Delete: + m.data.Delete(op.Key) + } + } + return nil +} + +func (m *memStorageClient) Close(_ context.Context) error { + return nil +} diff --git a/receiver/akamaisiemreceiver/metadata.yaml b/receiver/akamaisiemreceiver/metadata.yaml new file mode 100644 index 000000000..3f1e699e8 --- /dev/null +++ b/receiver/akamaisiemreceiver/metadata.yaml @@ -0,0 +1,142 @@ +type: akamai_siem +scope_name: github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver +github_project: elastic/opentelemetry-collector-components + +status: + class: receiver + stability: + alpha: [logs] + +tests: + config: + +telemetry: + metrics: + akamai_siem.bytes_received: + enabled: true + description: Total bytes received from the Akamai SIEM API + unit: By + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.cursor_persists: + enabled: true + description: Total successful cursor persist operations + unit: "{persists}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.events_emitted: + enabled: true + description: Total events successfully forwarded to the downstream consumer via ConsumeLogs + unit: "{events}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.events_per_page: + enabled: true + description: Number of events received per API response page + unit: "{events}" + stability: alpha + histogram: + value_type: int + bucket_boundaries: [ 0, 10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, 600000 ] + akamai_siem.events_per_second: + enabled: true + description: Events per second throughput per poll cycle + unit: "{events}/s" + stability: alpha + histogram: + value_type: double + bucket_boundaries: [ 100, 500, 1000, 5000, 10000, 25000, 50000, 100000, 500000 ] + akamai_siem.events_received: + enabled: true + description: Total events received from the Akamai SIEM API + unit: "{events}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.invalid_timestamp_retries: + enabled: true + description: Total HMAC timestamp retries + unit: "{retries}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.offset_expired: + enabled: true + description: Total 416 offset out of range errors + unit: "{errors}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.offset_ttl_drops: + enabled: true + description: Total proactive offset TTL expirations + unit: "{drops}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.page_processing_time: + enabled: true + description: Time to process a page (NDJSON parse + body-map construction + ConsumeLogs) + unit: s + stability: alpha + histogram: + value_type: double + bucket_boundaries: [ 0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 10.0 ] + akamai_siem.pages_processed: + enabled: true + description: Total API response pages processed + unit: "{pages}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.poll_duration: + enabled: true + description: Duration of a full poll iteration (may include multiple pages) + unit: s + stability: alpha + histogram: + value_type: double + bucket_boundaries: [ 0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0 ] + akamai_siem.recovery_attempts: + enabled: true + description: Total recovery actions taken (416 replays, timestamp retries, from clamps) + unit: "{attempts}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.request_duration: + enabled: true + description: Duration of API requests to the Akamai SIEM API + unit: s + stability: alpha + histogram: + value_type: double + bucket_boundaries: [ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0 ] + akamai_siem.request_errors: + enabled: true + description: Total failed API requests (non-200 responses) + unit: "{requests}" + stability: alpha + sum: + value_type: int + monotonic: true + akamai_siem.requests: + enabled: true + description: Total API requests made to the Akamai SIEM API + unit: "{requests}" + stability: alpha + sum: + value_type: int + monotonic: true diff --git a/receiver/akamaisiemreceiver/receiver.go b/receiver/akamaisiemreceiver/receiver.go new file mode 100644 index 000000000..55df44a25 --- /dev/null +++ b/receiver/akamaisiemreceiver/receiver.go @@ -0,0 +1,318 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver // import "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver" + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/extension/xextension/storage" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" + + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/akamaiclient" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/auth" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/cursor" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/metadata" + "github.com/elastic/opentelemetry-collector-components/receiver/akamaisiemreceiver/internal/poller" +) + +// akamaiReceiver polls the Akamai SIEM API and emits logs as plog.Logs with +// raw JSON in a body map keyed "message", along with data_stream.* fields and +// the elastic.mapping.mode: bodymap scope attribute. The Elasticsearch exporter +// serializes the body map directly into the document. +type akamaiReceiver struct { + cfg *Config + settings receiver.Settings + log *zap.Logger + consumer consumer.Logs + + cancel context.CancelFunc + wg sync.WaitGroup + tracer trace.Tracer // nil-safe +} + +func newAkamaiReceiver(cfg *Config, settings receiver.Settings, cons consumer.Logs) (*akamaiReceiver, error) { + return &akamaiReceiver{ + cfg: cfg, + settings: settings, + log: settings.Logger, + consumer: cons, + }, nil +} + +// Start implements receiver.Logs. +func (r *akamaiReceiver) Start(ctx context.Context, host component.Host) error { + // Create cursor store for state persistence via storage extension. + var cursorStore *cursor.CursorStore + if r.cfg.StorageID != nil { + storageClient, err := getStorageClient(ctx, host, r.cfg.StorageID, r.settings.ID) + if err != nil { + return fmt.Errorf("failed to get storage client: %w", err) + } + cursorStore = cursor.NewCursorStore(storageClient) + } + + // Load persisted cursor. + var cur cursor.Cursor + if cursorStore != nil { + var err error + cur, err = cursorStore.Load(ctx) + if err != nil { + r.log.Warn("failed to load cursor, starting fresh", zap.Error(err)) + cur = cursor.Cursor{} + } else if cur.ChainFrom != 0 { + r.log.Info("loaded persisted cursor", + zap.Int64("chain_from", cur.ChainFrom), + zap.Int64("chain_to", cur.ChainTo), + zap.Bool("caught_up", cur.CaughtUp), + zap.String("last_offset", cur.LastOffset), + ) + } + } + + // Create HTTP client from confighttp.ClientConfig (handles TLS, proxy, timeout). + httpClient, err := r.cfg.HTTP.ToClient(ctx, host.GetExtensions(), r.settings.TelemetrySettings) + if err != nil { + return fmt.Errorf("failed to create HTTP client: %w", err) + } + + // Wrap transport with EdgeGrid signing. ToClient may return a client with a + // nil Transport (meaning use http.DefaultTransport); guard against that so + // auth.Transport.RoundTrip never dereferences a nil Base. + signer := auth.NewEdgeGridSigner( + string(r.cfg.Authentication.ClientToken), + string(r.cfg.Authentication.ClientSecret), + string(r.cfg.Authentication.AccessToken), + ) + base := httpClient.Transport + if base == nil { + base = http.DefaultTransport + } + httpClient.Transport = &auth.Transport{ + Base: base, + Signer: signer, + } + + client, err := akamaiclient.NewClient( + httpClient, + r.cfg.HTTP.Endpoint, + r.cfg.ConfigIDs, + r.log, + ) + if err != nil { + return err + } + + pollerCfg := poller.PollerConfig{ + EventLimit: r.cfg.EventLimit, + InitialLookback: r.cfg.InitialLookback, + OffsetTTL: r.cfg.OffsetTTL, + MaxRecoveryAttempts: r.cfg.MaxRecoveryAttempts, + InvalidTimestampRetries: r.cfg.InvalidTimestampRetries, + BatchSize: r.cfg.BatchSize, + StreamBufferSize: r.cfg.StreamBufferSize, + } + + // Build telemetry instruments. + tb, err := metadata.NewTelemetryBuilder(r.settings.TelemetrySettings) + if err != nil { + r.log.Warn("failed to create telemetry builder, metrics disabled", zap.Error(err)) + } + var tel *poller.Telemetry + if tb != nil { + r.tracer = metadata.Tracer(r.settings.TelemetrySettings) + tel = &poller.Telemetry{ + Tracer: r.tracer, + Requests: tb.AkamaiSiemRequests, + RequestErrors: tb.AkamaiSiemRequestErrors, + EventsReceived: tb.AkamaiSiemEventsReceived, + EventsEmitted: tb.AkamaiSiemEventsEmitted, + OffsetExpired: tb.AkamaiSiemOffsetExpired, + OffsetTTLDrops: tb.AkamaiSiemOffsetTTLDrops, + RecoveryAttempts: tb.AkamaiSiemRecoveryAttempts, + InvalidTSRetries: tb.AkamaiSiemInvalidTimestampRetries, + RequestDuration: tb.AkamaiSiemRequestDuration, + PollDuration: tb.AkamaiSiemPollDuration, + EventsPerSecond: tb.AkamaiSiemEventsPerSecond, + PagesProcessed: tb.AkamaiSiemPagesProcessed, + CursorPersists: tb.AkamaiSiemCursorPersists, + BytesReceived: tb.AkamaiSiemBytesReceived, + PageProcessingTime: tb.AkamaiSiemPageProcessingTime, + + EventsPerPage: tb.AkamaiSiemEventsPerPage, + } + } + + poll := poller.NewPoller(client, cursorStore, cur, pollerCfg, r.emitEvents, r.log, tel) + + pollCtx, cancel := context.WithCancel(context.Background()) + r.cancel = cancel + + r.wg.Add(1) + go func() { + defer r.wg.Done() + defer client.Close() + if cursorStore != nil { + defer func() { _ = cursorStore.Close(context.Background()) }() + } + r.pollLoop(pollCtx, poll) + }() + + storageInfo := "disabled" + if r.cfg.StorageID != nil { + storageInfo = r.cfg.StorageID.String() + } + r.log.Info("akamai SIEM receiver started", + zap.String("endpoint", r.cfg.HTTP.Endpoint), + zap.String("config_ids", r.cfg.ConfigIDs), + zap.Duration("poll_interval", r.cfg.PollInterval), + zap.Int("event_limit", r.cfg.EventLimit), + zap.String("storage", storageInfo), + ) + + return nil +} + +// Shutdown implements receiver.Logs. It respects the context deadline so a +// hung poll goroutine does not block the collector's shutdown indefinitely. +func (r *akamaiReceiver) Shutdown(ctx context.Context) error { + r.log.Info("akamai SIEM receiver shutting down") + if r.cancel != nil { + r.cancel() + } + done := make(chan struct{}) + go func() { + r.wg.Wait() + close(done) + }() + select { + case <-done: + r.log.Info("akamai SIEM receiver stopped") + return nil + case <-ctx.Done(): + r.log.Warn("akamai SIEM receiver shutdown context expired", zap.Error(ctx.Err())) + return ctx.Err() + } +} + +func (r *akamaiReceiver) pollLoop(ctx context.Context, poll *poller.Poller) { + // Run first poll immediately. + if err := poll.Poll(ctx); err != nil { + if ctx.Err() != nil { + return + } + r.log.Error("poll failed", zap.Error(err)) + } + + ticker := time.NewTicker(r.cfg.PollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := poll.Poll(ctx); err != nil { + if ctx.Err() != nil { + return + } + r.log.Error("poll failed", zap.Error(err)) + } + } + } +} + +// emitEvents converts raw JSON event strings to plog.Logs and forwards them +// to the configured consumer. +// +// Each log record carries the raw Akamai JSON in a body map keyed "message", +// alongside data_stream.{type,dataset,namespace} body keys for Kibana filters. +// data_stream.* is also written to the resource attributes so the Elasticsearch +// exporter's dynamic routing can target the correct data stream. The +// elastic.mapping.mode: bodymap scope attribute tells the ES exporter to +// serialize the body map fields directly into the indexed document. +func (r *akamaiReceiver) emitEvents(ctx context.Context, events []string) error { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + sl := rl.ScopeLogs().AppendEmpty() + + // Resource attributes — used by the ES exporter's dynamic routing for + // data-stream targeting. Bodymap mode does NOT serialize resource attrs + // into the document, so these data_stream.* values are also written into + // the body map below for Kibana filters. + rattr := rl.Resource().Attributes() + rattr.PutStr("data_stream.type", r.cfg.DataStream.Type) + rattr.PutStr("data_stream.dataset", r.cfg.DataStream.Dataset) + rattr.PutStr("data_stream.namespace", r.cfg.DataStream.Namespace) + + // Scope attribute — tells the Elasticsearch exporter to use bodymap mode, + // which serializes the body map fields directly into the document. + sl.Scope().Attributes().PutStr("elastic.mapping.mode", "bodymap") + + now := pcommon.NewTimestampFromTime(time.Now()) + + if r.tracer != nil { + var span trace.Span + ctx, span = r.tracer.Start(ctx, "akamai_siem.EmitEvents", + trace.WithAttributes(attribute.Int("event_count", len(events))), + ) + defer span.End() + } + + for _, rawJSON := range events { + lr := sl.LogRecords().AppendEmpty() + lr.SetTimestamp(now) + lr.SetObservedTimestamp(now) + body := lr.Body().SetEmptyMap() + body.PutStr("message", rawJSON) + // Body data_stream.* — bodymap mode serializes only body content into + // the indexed document, so these are needed for Kibana filters + // (data_stream.dataset:akamai.siem etc.) to match. + body.PutStr("data_stream.type", r.cfg.DataStream.Type) + body.PutStr("data_stream.dataset", r.cfg.DataStream.Dataset) + body.PutStr("data_stream.namespace", r.cfg.DataStream.Namespace) + } + + return r.consumer.ConsumeLogs(ctx, logs) +} + +// getStorageClient retrieves a storage.Client from the configured storage extension. +func getStorageClient(ctx context.Context, host component.Host, id *component.ID, componentID component.ID) (storage.Client, error) { + if id == nil { + return nil, fmt.Errorf("storage extension ID is nil") + } + ext, ok := host.GetExtensions()[*id] + if !ok { + return nil, fmt.Errorf("storage extension %q not found", id) + } + se, ok := ext.(storage.Extension) + if !ok { + return nil, fmt.Errorf("extension %q is not a storage extension", id) + } + return se.GetClient(ctx, component.KindReceiver, componentID, "") +} diff --git a/receiver/akamaisiemreceiver/receiver_test.go b/receiver/akamaisiemreceiver/receiver_test.go new file mode 100644 index 000000000..15eed572a --- /dev/null +++ b/receiver/akamaisiemreceiver/receiver_test.go @@ -0,0 +1,741 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/extension/xextension/storage" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +func TestFactory_Type(t *testing.T) { + f := NewFactory() + assert.Equal(t, "akamai_siem", f.Type().String()) +} + +func TestFactory_CreateDefaultConfig(t *testing.T) { + f := NewFactory() + cfg := f.CreateDefaultConfig() + require.NotNil(t, cfg) + assert.NoError(t, componenttest.CheckConfigStruct(cfg)) +} + +func TestReceiver_StartShutdown(t *testing.T) { + // Mock Akamai API that returns empty response. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "12345" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour // don't poll again during test + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + + err = rcv.Start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) + + // Give the first poll time to execute. + time.Sleep(500 * time.Millisecond) + + err = rcv.Shutdown(context.Background()) + require.NoError(t, err) +} + +func TestReceiver_EmitsEvents(t *testing.T) { + // Mock Akamai API returning 2 events + offset context. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.Header.Get("Authorization"), "EG1-HMAC-SHA256") + _, _ = fmt.Fprintln(w, `{"attackData":{"rule":"950004"},"httpMessage":{"host":"example.com"}}`) + _, _ = fmt.Fprintln(w, `{"attackData":{"rule":"990011"},"httpMessage":{"host":"test.com"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"cursor-abc","total":2,"limit":10000}`) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "12345" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + cfg.HTTP.Timeout = 10 * time.Second + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + + err = rcv.Start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) + + // Wait for poll to emit events. + assert.Eventually(t, func() bool { + return sink.LogRecordCount() >= 2 + }, 5*time.Second, 100*time.Millisecond, "expected at least 2 log records") + + err = rcv.Shutdown(context.Background()) + require.NoError(t, err) + + // Verify log bodies contain raw JSON. + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + rl := allLogs[0].ResourceLogs().At(0) + lr := rl.ScopeLogs().At(0).LogRecords().At(0) + assert.Contains(t, bodyMessage(t, lr), `"rule":"950004"`) +} + +// TestReceiver_CursorPersistAndResume verifies that: +// 1. First run fetches with time-based params (no cursor), receives events, persists cursor with offset +// 2. Cursor is written to the storage extension with correct content +// 3. Second run loads cursor and resumes with offset-based fetch (not time-based) +func TestReceiver_CursorPersistAndResume(t *testing.T) { + var requests []map[string]string + var mu sync.Mutex + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, map[string]string{ + "offset": r.URL.Query().Get("offset"), + "from": r.URL.Query().Get("from"), + "to": r.URL.Query().Get("to"), + }) + mu.Unlock() + + // Return 1 event (less than limit=10000 → chain drained, caught_up=true). + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"saved-cursor-abc","total":1,"limit":10000}`) + })) + defer server.Close() + + memClient := newMemStorageClient() + storageID := component.MustNewID("file_storage") + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "99" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + cfg.StorageID = &storageID + + set := receivertest.NewNopSettings(NewFactory().Type()) + host := mockHost(memClient) + + // --- First run: should use time-based fetch (no cursor exists yet) --- + sink1 := &consumertest.LogsSink{} + rcv1, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink1) + require.NoError(t, err) + require.NoError(t, rcv1.Start(context.Background(), host)) + require.Eventually(t, func() bool { return sink1.LogRecordCount() >= 1 }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, rcv1.Shutdown(context.Background())) + + // Verify first request was time-based (has "from" and "to", no "offset"). + mu.Lock() + require.NotEmpty(t, requests, "first run should have made at least one request") + firstReq := requests[0] + mu.Unlock() + assert.NotEmpty(t, firstReq["from"], "first run should use time-based fetch (from)") + assert.NotEmpty(t, firstReq["to"], "first run should use time-based fetch (to)") + assert.Empty(t, firstReq["offset"], "first run should NOT have offset (no cursor)") + + // Verify cursor was written to storage extension. + cursorData, err := memClient.Get(context.Background(), "akamai_siem_cursor") + require.NoError(t, err, "cursor should exist in storage after first run") + require.NotNil(t, cursorData, "cursor data should not be nil after first run") + + var savedCursor map[string]any + require.NoError(t, json.Unmarshal(cursorData, &savedCursor)) + assert.Equal(t, "saved-cursor-abc", savedCursor["last_offset"], "cursor should contain the offset from API response") + assert.NotZero(t, savedCursor["chain_from"], "cursor should have chain_from") + assert.NotZero(t, savedCursor["chain_to"], "cursor should have chain_to") + assert.True(t, savedCursor["caught_up"].(bool), "cursor should be caught_up (events < limit)") + + // --- Second run: should resume from cursor with offset-based fetch --- + mu.Lock() + requests = nil // Reset request log. + mu.Unlock() + + sink2 := &consumertest.LogsSink{} + rcv2, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink2) + require.NoError(t, err) + require.NoError(t, rcv2.Start(context.Background(), host)) + require.Eventually(t, func() bool { return sink2.LogRecordCount() >= 1 }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, rcv2.Shutdown(context.Background())) + + // Second run: cursor was caught_up=true, so it starts a NEW chain (Branch 3), + // not an offset-based fetch. This is correct — caught_up means the previous + // chain was fully drained, so we start fresh from chain_to with overlap. + mu.Lock() + require.NotEmpty(t, requests, "second run should have made at least one request") + secondReq := requests[0] + mu.Unlock() + + // Branch 3 (new chain) uses from/to, not offset. + // The "from" should be based on the previous chain_to minus overlap. + assert.NotEmpty(t, secondReq["from"], "second run should start a new chain with from") + assert.NotEmpty(t, secondReq["to"], "second run should start a new chain with to") +} + +// TestReceiver_CursorResume_OffsetDrain verifies that when the cursor has +// caught_up=false + valid offset, the second run resumes with offset-based fetch. +func TestReceiver_CursorResume_OffsetDrain(t *testing.T) { + // Pre-seed a cursor in the mock storage that simulates an interrupted chain drain. + memClient := newMemStorageClient() + cursor := map[string]any{ + "chain_from": time.Now().Add(-1 * time.Hour).Unix(), + "chain_to": time.Now().Add(-1 * time.Minute).Unix(), + "caught_up": false, + "last_offset": "resume-from-this-offset", + "offset_obtained_at": time.Now().Add(-10 * time.Second).Format(time.RFC3339Nano), + } + cursorData, _ := json.Marshal(cursor) + require.NoError(t, memClient.Set(context.Background(), "akamai_siem_cursor", cursorData)) + + storageID := component.MustNewID("file_storage") + + var capturedOffset string + var mu sync.Mutex + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + capturedOffset = r.URL.Query().Get("offset") + mu.Unlock() + // Return fewer events than limit → chain drained. + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"next-offset","total":1,"limit":10000}`) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "99" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + cfg.StorageID = &storageID + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), mockHost(memClient))) + require.Eventually(t, func() bool { return sink.LogRecordCount() >= 1 }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + // The receiver should have resumed with the pre-seeded offset (Branch 1: drain). + mu.Lock() + assert.Equal(t, "resume-from-this-offset", capturedOffset, + "receiver should resume from the persisted offset") + mu.Unlock() +} + +func TestReceiver_TestdataResponse(t *testing.T) { + // Serve the realistic NDJSON testdata file through a mock server. + ndjson, err := os.ReadFile("testdata/siem_response.ndjson") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(ndjson) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "67217" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + + assert.Eventually(t, func() bool { + return sink.LogRecordCount() >= 2 + }, 5*time.Second, 100*time.Millisecond, "expected 2 events from testdata") + + require.NoError(t, rcv.Shutdown(context.Background())) + + // Verify the events contain real Akamai SIEM fields. + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + records := allLogs[0].ResourceLogs().At(0).ScopeLogs().At(0).LogRecords() + assert.Equal(t, 2, records.Len()) + + body0 := bodyMessage(t, records.At(0)) + assert.Contains(t, body0, `"clientIP":"198.51.100.1"`) + assert.Contains(t, body0, `"configId":"67217"`) + assert.Contains(t, body0, `"host":"example.com"`) + + body1 := bodyMessage(t, records.At(1)) + assert.Contains(t, body1, `"clientIP":"203.0.113.42"`) + assert.Contains(t, body1, `"status":"403"`) +} + +func TestReceiver_BodyIsRawJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"x","total":1,"limit":10000}`) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + + assert.Eventually(t, func() bool { + return sink.LogRecordCount() >= 1 + }, 5*time.Second, 100*time.Millisecond) + + require.NoError(t, rcv.Shutdown(context.Background())) + + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + rl := allLogs[0].ResourceLogs().At(0) + + lr := rl.ScopeLogs().At(0).LogRecords().At(0) + assert.Equal(t, 0, lr.Attributes().Len(), "log record should have no attributes") + + // Body is a map with raw JSON in "message" key. + assert.Contains(t, bodyMessage(t, lr), `"host":"test.com"`) +} + +// TestReceiver_FullFlow tests the complete poll flow: +// Mock Akamai API → EdgeGrid auth → NDJSON parse → cursor persist +// → plog.Logs with body map {message: rawJSON, data_stream.*}. +func TestReceiver_FullFlow(t *testing.T) { + var requestCount atomic.Int32 + ndjson, err := os.ReadFile("testdata/siem_response.ndjson") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount.Add(1) + auth := r.Header.Get("Authorization") + require.Contains(t, auth, "EG1-HMAC-SHA256", "request missing EdgeGrid auth") + require.Contains(t, auth, "client_token=ct") + require.Contains(t, auth, "access_token=at") + require.Contains(t, auth, "signature=") + assert.Equal(t, "/siem/v1/configs/12345", r.URL.Path) + assert.NotEmpty(t, r.URL.Query().Get("limit")) + _, _ = w.Write(ndjson) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "12345" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + + require.Eventually(t, func() bool { + return sink.LogRecordCount() >= 2 + }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + require.Equal(t, 1, allLogs[0].ResourceLogs().Len()) + rl := allLogs[0].ResourceLogs().At(0) + + require.Equal(t, 1, rl.ScopeLogs().Len()) + records := rl.ScopeLogs().At(0).LogRecords() + assert.Equal(t, 2, records.Len(), "expected 2 events (offset context excluded)") + + for i := 0; i < records.Len(); i++ { + lr := records.At(i) + body := bodyMessage(t, lr) + assert.True(t, json.Valid([]byte(body)), "record %d body should be valid JSON", i) + assert.Contains(t, body, `"attackData"`, "record %d missing attackData", i) + assert.Contains(t, body, `"httpMessage"`, "record %d missing httpMessage", i) + assert.Greater(t, int64(lr.Timestamp()), int64(0), "record %d timestamp not set", i) + assert.Greater(t, int64(lr.ObservedTimestamp()), int64(0), "record %d observed timestamp not set", i) + assert.Equal(t, 0, lr.Attributes().Len(), "record %d should have no attributes", i) + } + + assert.Contains(t, bodyMessage(t, records.At(0)), `"clientIP":"198.51.100.1"`) + assert.Contains(t, bodyMessage(t, records.At(1)), `"clientIP":"203.0.113.42"`) + + // Offset context should not appear as an event. + for i := 0; i < records.Len(); i++ { + assert.NotContains(t, bodyMessage(t, records.At(i)), `"offset"`) + } + + assert.GreaterOrEqual(t, int(requestCount.Load()), 1) +} + +// TestReceiver_ChainDrain_MultiPage verifies multi-page chain draining: +// page 1 returns events == limit → fetches page 2 → fewer events → drained. +func TestReceiver_ChainDrain_MultiPage(t *testing.T) { + var pageRequests atomic.Int32 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := int(pageRequests.Add(1)) + switch page { + case 1: + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"page1.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1001","host":"page1.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"page2-cursor","total":2,"limit":2}`) + case 2: + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1002","host":"page2.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"final","total":1,"limit":2}`) + default: + w.WriteHeader(200) + } + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.EventLimit = 2 + cfg.PollInterval = 24 * time.Hour + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + require.Eventually(t, func() bool { return sink.LogRecordCount() >= 3 }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + assert.GreaterOrEqual(t, int(pageRequests.Load()), 2, "should fetch at least 2 pages") + assert.Equal(t, 3, sink.LogRecordCount()) +} + +// TestReceiver_BatchedEmission verifies that a page with more events than +// batch_size is split into multiple ConsumeLogs calls. +func TestReceiver_BatchedEmission(t *testing.T) { + // Generate 5 events. With batch_size=2, expect 3 ConsumeLogs calls (2+2+1). + var sb strings.Builder + for i := 0; i < 5; i++ { + _, _ = fmt.Fprintf(&sb, `{"httpMessage":{"start":"%d","host":"batch.com","status":"200"}}%s`, 1000+i, "\n") + } + _, _ = fmt.Fprintln(&sb, `{"offset":"x","total":5,"limit":10000}`) + ndjson := sb.String() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, ndjson) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + cfg.BatchSize = 2 + cfg.StreamBufferSize = 2 + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + + require.Eventually(t, func() bool { + return sink.LogRecordCount() >= 5 + }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + // Total events should be 5. + assert.Equal(t, 5, sink.LogRecordCount()) + + // With batch_size=2, we expect 3 separate ConsumeLogs calls → 3 plog.Logs entries. + allLogs := sink.AllLogs() + assert.GreaterOrEqual(t, len(allLogs), 3, + "5 events with batch_size=2 should produce at least 3 ConsumeLogs calls, got %d", len(allLogs)) +} + +// TestReceiver_EmitFailure_NoCursorPersist verifies that if ConsumeLogs fails +// mid-page, the cursor is NOT persisted (events not confirmed). +func TestReceiver_EmitFailure_NoCursorPersist(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"should-not-persist","total":1,"limit":10000}`) + })) + defer server.Close() + + memClient := newMemStorageClient() + storageID := component.MustNewID("file_storage") + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + cfg.StorageID = &storageID + + // Use a consumer that always fails. + failConsumer := &failingConsumer{} + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := newAkamaiReceiver(cfg, set, failConsumer) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), mockHost(memClient))) + + // Give time for one poll to execute and fail. + time.Sleep(1 * time.Second) + require.NoError(t, rcv.Shutdown(context.Background())) + + // Cursor should NOT have been persisted because ConsumeLogs failed. + cursorData, err := memClient.Get(context.Background(), "akamai_siem_cursor") + require.NoError(t, err) + assert.Nil(t, cursorData, "cursor should NOT be persisted when ConsumeLogs fails") +} + +// failingConsumer is a consumer.Logs that always returns an error. +type failingConsumer struct{} + +func (f *failingConsumer) ConsumeLogs(_ context.Context, _ plog.Logs) error { + return fmt.Errorf("simulated ConsumeLogs failure") +} +func (f *failingConsumer) Capabilities() consumer.Capabilities { + return consumer.Capabilities{} +} + +// --- test helpers --- + +// TestEmitEvents_BodyShape verifies that emitEvents produces a body map with +// a "message" key containing the raw JSON string and data_stream.* body keys +// for Kibana filters. +func TestEmitEvents_BodyShape(t *testing.T) { + cfg := createDefaultConfig().(*Config) + sink := &consumertest.LogsSink{} + rcv, err := newAkamaiReceiver(cfg, receivertest.NewNopSettings(NewFactory().Type()), sink) + require.NoError(t, err) + + err = rcv.emitEvents(context.Background(), []string{`{"test":"event"}`}) + require.NoError(t, err) + + allLogs := sink.AllLogs() + require.NotEmpty(t, allLogs) + lr := allLogs[0].ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) + + require.Equal(t, pcommon.ValueTypeMap, lr.Body().Type(), "body should be a map") + body := lr.Body().Map() + + v, ok := body.Get("message") + require.True(t, ok, "body map should have 'message' key") + assert.Equal(t, `{"test":"event"}`, v.Str()) + + for k, want := range map[string]string{ + "data_stream.type": "logs", + "data_stream.dataset": "akamai.siem", + "data_stream.namespace": "default", + } { + got, ok := body.Get(k) + require.True(t, ok, "body map should have %q key", k) + assert.Equal(t, want, got.Str(), "body[%q]", k) + } +} + +// TestEmitEvents_ScopeAndResource verifies that emitEvents sets the bodymap +// mapping mode on the scope and writes data_stream.* to resource attributes +// for ES exporter routing. +func TestEmitEvents_ScopeAndResource(t *testing.T) { + cfg := createDefaultConfig().(*Config) + cfg.DataStream.Dataset = "akamai.siem.tenant_a" + sink := &consumertest.LogsSink{} + rcv, err := newAkamaiReceiver(cfg, receivertest.NewNopSettings(NewFactory().Type()), sink) + require.NoError(t, err) + + err = rcv.emitEvents(context.Background(), []string{`{"x":1}`}) + require.NoError(t, err) + + rl := sink.AllLogs()[0].ResourceLogs().At(0) + rattr := rl.Resource().Attributes() + for k, want := range map[string]string{ + "data_stream.type": "logs", + "data_stream.dataset": "akamai.siem.tenant_a", + "data_stream.namespace": "default", + } { + got, ok := rattr.Get(k) + require.True(t, ok, "resource attr %q missing", k) + assert.Equal(t, want, got.Str(), "resource attr %q", k) + } + + scope := rl.ScopeLogs().At(0).Scope() + mode, ok := scope.Attributes().Get("elastic.mapping.mode") + require.True(t, ok, "scope attribute elastic.mapping.mode missing") + assert.Equal(t, "bodymap", mode.Str()) +} + +// bodyMessage extracts the "message" string from a LogRecord body map. +func bodyMessage(t *testing.T, lr plog.LogRecord) string { + t.Helper() + require.Equal(t, pcommon.ValueTypeMap, lr.Body().Type(), "body should be a map") + v, ok := lr.Body().Map().Get("message") + require.True(t, ok, "body map should have 'message' key") + return v.Str() +} + +// mockStorageExtension implements storage.Extension for tests. +type mockStorageExtension struct { + component.StartFunc + component.ShutdownFunc + client *memStorageClient +} + +func (m *mockStorageExtension) GetClient(_ context.Context, _ component.Kind, _ component.ID, _ string) (storage.Client, error) { + return m.client, nil +} + +type memStorageClient struct { + data sync.Map +} + +func newMemStorageClient() *memStorageClient { + return &memStorageClient{} +} + +func (m *memStorageClient) Get(_ context.Context, key string) ([]byte, error) { + v, ok := m.data.Load(key) + if !ok { + return nil, nil + } + return v.([]byte), nil +} +func (m *memStorageClient) Set(_ context.Context, key string, value []byte) error { + m.data.Store(key, value) + return nil +} +func (m *memStorageClient) Delete(_ context.Context, key string) error { + m.data.Delete(key) + return nil +} +func (m *memStorageClient) Batch(_ context.Context, ops ...*storage.Operation) error { + for _, op := range ops { + switch op.Type { + case storage.Get: + v, _ := m.data.Load(op.Key) + if v != nil { + op.Value = v.([]byte) + } + case storage.Set: + m.data.Store(op.Key, op.Value) + case storage.Delete: + m.data.Delete(op.Key) + } + } + return nil +} +func (m *memStorageClient) Close(_ context.Context) error { return nil } + +// mockHost returns a component.Host with a mock storage extension registered. +func mockHost(storageClient *memStorageClient) component.Host { + ext := &mockStorageExtension{client: storageClient} + return &mockHostImpl{extensions: map[component.ID]component.Component{ + component.MustNewID("file_storage"): ext, + }} +} + +type mockHostImpl struct { + component.Host + extensions map[component.ID]component.Component +} + +func (h *mockHostImpl) GetExtensions() map[component.ID]component.Component { + return h.extensions +} diff --git a/receiver/akamaisiemreceiver/testdata/siem_response.ndjson b/receiver/akamaisiemreceiver/testdata/siem_response.ndjson new file mode 100644 index 000000000..eed7991bc --- /dev/null +++ b/receiver/akamaisiemreceiver/testdata/siem_response.ndjson @@ -0,0 +1,3 @@ +{"attackData":{"appliedAction":"tarpit","clientIP":"198.51.100.1","configId":"67217","policyId":"PNWD_110088","ruleActions":"bW9uaXRvcg%3d%3d%3bbW9uaXRvcg%3d%3d","ruleData":"%3b%3b","ruleMessages":"TWlzc2luZyBDb29raWUgSGVhZGVy%3bTm9uLVBlcnNpc3RlbnQgSFRUUCBDb25uZWN0aW9u","rules":"MzkwNDAwNg%3d%3d%3bMzkwNDAwNw%3d%3d"},"format":"json","geo":{"asn":"28573","city":"SOROCABA","continent":"SA","country":"BR","regionCode":"SP"},"httpMessage":{"bytes":"0","host":"example.com","method":"GET","path":"/api/test","port":"443","protocol":"HTTP/1.1","query":"q=test","requestId":"f3fe4c34","start":"1762365006","status":"200","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"} +{"attackData":{"appliedAction":"monitor","clientIP":"203.0.113.42","configId":"67217","policyId":"PNWD_110088","ruleActions":"bW9uaXRvcg%3d%3d","ruleData":"","ruleMessages":"Q2hyb21lIFNpZ25hdHVyZSBBbm9tYWx5","rules":"MzkwNDAyMA%3d%3d"},"format":"json","geo":{"asn":"15169","city":"MOUNTAIN VIEW","continent":"NA","country":"US","regionCode":"CA"},"httpMessage":{"bytes":"1234","host":"test.example.com","method":"POST","path":"/login","port":"443","protocol":"HTTP/2","query":"","requestId":"a1b2c3d4","start":"1762365010","status":"403","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"} +{"offset":"eyJhbGciOiJIUzI1NiJ9.next-page-cursor","total":2,"limit":10000} diff --git a/receiver/akamaisiemreceiver/testdata/siem_response_full.ndjson b/receiver/akamaisiemreceiver/testdata/siem_response_full.ndjson new file mode 100644 index 000000000..df71d95d0 --- /dev/null +++ b/receiver/akamaisiemreceiver/testdata/siem_response_full.ndjson @@ -0,0 +1,4 @@ +{"attackData":{"appliedAction":"deny","clientIP":"198.51.100.1","configId":"67217","policyId":"PNWD_110088","ruleActions":"YWxlcnQ%3D%3BZGVueQ%3D%3D","ruleData":"c3VzcGljaW91c192YWx1ZV85%3Bc3VzcGljaW91c192YWx1ZV8xOQ%3D%3D","ruleMessages":"U1FMIEluamVjdGlvbiBhdHRlbXB0IGRldGVjdGVk%3BQ3Jvc3MtU2l0ZSBTY3JpcHRpbmcgYXR0ZW1wdCBkZXRlY3RlZA%3D%3D","ruleSelectors":"QVJHUzpwYXJhbQ%3D%3D%3BQVJHUzpwYXJhbQ%3D%3D","ruleTags":"U1FM%3BYHNZ","ruleVersions":"NA%3D%3D%3BNw%3D%3D","rules":"OTUwMDAy%3BOTUwMDE5"},"format":"json","geo":{"asn":"15169","city":"MOUNTAIN VIEW","continent":"NA","country":"US","regionCode":"CA"},"httpMessage":{"bytes":"4096","host":"api.example.com","method":"POST","path":"/v2/users","port":"443","protocol":"HTTP/1.1","query":"id=1234&action=update","requestHeaders":"User-Agent: Mozilla/5.0\r\nAccept: application/json\r\nHost: api.example.com","requestId":"abc123def456","responseHeaders":"Content-Type: application/json\r\nX-Request-Id: abc123def456","start":"1700000000","status":"403","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"} +{"attackData":{"appliedAction":"monitor","clientIP":"203.0.113.42","configId":"67217","policyId":"PNWD_110088","ruleActions":"bW9uaXRvcg%3D%3D","ruleData":"","ruleMessages":"TWlzc2luZyBDb29raWUgSGVhZGVy","rules":"MzkwNDAwNg%3D%3D","ruleSelectors":"","ruleTags":"T1dBU1A%3D","ruleVersions":"MQ%3D%3D"},"format":"json","geo":{"asn":"28573","city":"SAO PAULO","continent":"SA","country":"BR","regionCode":"SP"},"httpMessage":{"bytes":"0","host":"www.example.com","method":"GET","path":"/health","port":"80","protocol":"HTTP/2.0","query":"","requestHeaders":"User-Agent: curl/7.68.0","requestId":"deadbeef0001","responseHeaders":"","start":"1700000005","status":"200"},"type":"akamai_siem","version":"1.0"} +{"attackData":{"appliedAction":"alert","clientIP":"10.0.0.1","configId":"67217","policyId":"TEST_POLICY","ruleActions":"YWxlcnQ%3D","ruleData":"dGVzdF9kYXRh","ruleMessages":"VGVzdCBydWxlIHRyaWdnZXJlZA%3D%3D","ruleSelectors":"QVJHUzp0ZXN0","ruleTags":"VEVTVA%3D%3D","ruleVersions":"MQ%3D%3D","rules":"OTk5OTk5"},"format":"json","geo":{"asn":"64496","city":"TESTVILLE","continent":"EU","country":"DE","regionCode":"BE"},"httpMessage":{"bytes":"256","host":"test.internal","method":"DELETE","path":"/api/v1/resource/42","port":"8443","protocol":"HTTP/1.1","query":"force=true","requestHeaders":"Authorization: Bearer token123","requestId":"test-req-001","responseHeaders":"X-Powered-By: Test","start":"1700000010","status":"500"},"type":"akamai_siem","version":"1.0"} +{"offset":"test-cursor-abc123","total":3,"limit":10000} diff --git a/receiver/akamaisiemreceiver/tracing_test.go b/receiver/akamaisiemreceiver/tracing_test.go new file mode 100644 index 000000000..88ae2a7d7 --- /dev/null +++ b/receiver/akamaisiemreceiver/tracing_test.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package akamaisiemreceiver + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver/receivertest" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +// TestTracing_SpansCreated verifies that the receiver creates the expected +// trace spans when a TracerProvider is configured. +func TestTracing_SpansCreated(t *testing.T) { + ndjson, err := os.ReadFile("testdata/siem_response.ndjson") + require.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(ndjson) + })) + defer server.Close() + + // Set up in-memory span exporter to capture spans. + spanExporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider( + sdktrace.WithSyncer(spanExporter), + ) + defer func() { _ = tp.Shutdown(context.Background()) }() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "12345" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + storageID := component.MustNewID("file_storage") + cfg.StorageID = &storageID + + memClient := newMemStorageClient() + + sink := &consumertest.LogsSink{} + set := receivertest.NewNopSettings(NewFactory().Type()) + // Inject our test TracerProvider. + set.TracerProvider = tp + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), mockHost(memClient))) + + require.Eventually(t, func() bool { + return sink.LogRecordCount() >= 2 + }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + // Force flush to ensure all spans are exported. + _ = tp.ForceFlush(context.Background()) + + spans := spanExporter.GetSpans() + require.NotEmpty(t, spans, "expected trace spans to be created") + + // Collect span names. + spanNames := make(map[string]int) + for _, s := range spans { + spanNames[s.Name]++ + } + + // Verify expected spans exist. + assert.Contains(t, spanNames, "akamai_siem.Poll", "Poll span missing") + assert.Contains(t, spanNames, "akamai_siem.FetchPage", "FetchPage span missing") + assert.Contains(t, spanNames, "akamai_siem.ProcessPage", "ProcessPage span missing") + assert.Contains(t, spanNames, "akamai_siem.EmitEvents", "EmitEvents span missing") + assert.Contains(t, spanNames, "akamai_siem.PersistCursor", "PersistCursor span missing") + + // Verify parent-child: FetchPage and ProcessPage should be children of Poll. + var pollSpanID string + for _, s := range spans { + if s.Name == "akamai_siem.Poll" { + pollSpanID = s.SpanContext.SpanID().String() + break + } + } + require.NotEmpty(t, pollSpanID, "Poll span not found") + + for _, s := range spans { + if s.Name == "akamai_siem.FetchPage" || s.Name == "akamai_siem.ProcessPage" || s.Name == "akamai_siem.PersistCursor" { + assert.Equal(t, pollSpanID, s.Parent.SpanID().String(), + "span %q should be a child of Poll", s.Name) + } + } +} + +// TestTracing_NoSpans_WhenDisabled verifies zero overhead when no +// TracerProvider is configured (default). +func TestTracing_NoSpans_WhenDisabled(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, `{"httpMessage":{"start":"1000","host":"test.com","status":"200"}}`) + _, _ = fmt.Fprintln(w, `{"offset":"x","total":1,"limit":10000}`) + })) + defer server.Close() + + cfg := createDefaultConfig().(*Config) + cfg.HTTP.Endpoint = server.URL + cfg.ConfigIDs = "1" + cfg.Authentication = EdgeGridAuth{ + ClientToken: configopaque.String("ct"), + ClientSecret: configopaque.String("cs"), + AccessToken: configopaque.String("at"), + } + cfg.PollInterval = 24 * time.Hour + + sink := &consumertest.LogsSink{} + // Default NopSettings — no TracerProvider configured. + set := receivertest.NewNopSettings(NewFactory().Type()) + + rcv, err := NewFactory().CreateLogs(context.Background(), set, cfg, sink) + require.NoError(t, err) + require.NoError(t, rcv.Start(context.Background(), componenttest.NewNopHost())) + require.Eventually(t, func() bool { return sink.LogRecordCount() >= 1 }, 10*time.Second, 50*time.Millisecond) + require.NoError(t, rcv.Shutdown(context.Background())) + + // No way to capture spans from nop provider — this test verifies no panics + // or errors occur when tracing is disabled (the default production path). +}