Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion distributions/tulip/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,45 @@ receivers:
protocols:
grpc: {}

processors:
memory_limiter:
check_interval: 1s
limit_mib: 1500
spike_limit_mib: 512

exporters:
nop:
otlp:
endpoint: otlp.example.com:4317
sending_queue:
enabled: true
queue_size: 1000
batch:
flush_timeout: 200ms
min_size: 8192
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
Comment thread
perebaj marked this conversation as resolved.
otlphttp:
endpoint: https://otlp.example.com/otlp
sending_queue:
enabled: true
queue_size: 1000
batch:
flush_timeout: 200ms
min_size: 8192
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s

service:
extensions: [ health_check, pprof ]
pipelines:
traces:
receivers: [ otlp ]
processors: [ ]
processors: [ memory_limiter ]
exporters: [ nop, otlp, otlphttp ]
56 changes: 30 additions & 26 deletions distributions/tulip/manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,49 +18,53 @@ dist:
module: github.com/ollygarden/tulip/tulip
name: tulip
description: OllyGarden Tulip - Enterprise-supported OpenTelemetry Collector distribution
version: 0.144.0
version: 0.150.0
output_path: ./_build
build_tags: "grpcnotrace"

# Extensions: Authentication, health checks, profiling
extensions:
- gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/oidcauthextension v0.145.0
- gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/oidcauthextension v0.150.0

# Receivers: Ingesting telemetry data
receivers:
- gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.145.0
- gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.145.0
- gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.150.0
- gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.150.0

# Exporters: Sending telemetry to backends
exporters:
- gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.145.0
- gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.145.0
- gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.145.0
- gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.145.0
- gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.150.0
- gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.150.0
- gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.150.0
- gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.150.0

# Processors: Data transformation and enrichment
# Note: batchprocessor was removed in the LTS May 2026 release.
# It has been deprecated upstream (PR #15046) and has a known data loss bug (#12443).
# Use exporter-level sending_queue + batch configuration instead.
# See docs/tulip-lts-may2026-upgrade-plan.md for migration details.
processors:
- gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.145.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.145.0
- gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.150.0
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.150.0

# Connectors: Pipeline routing
connectors:
- gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.145.0
- gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.150.0

# Providers: Configuration sources
providers:
- gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.51.0
- gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.51.0
- gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.51.0
- gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.56.0
- gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.56.0
- gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.56.0
187 changes: 187 additions & 0 deletions docs/tulip-lts-may2026-upgrade-plan.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# Tulip LTS May 2026 — Upgrade Plan

## What is an LTS release?

LTS (Long-Term Support) is a release model where a specific version receives extended maintenance, security patches, and bug fixes over a longer period than regular releases. Unlike the upstream OpenTelemetry Collector, which ships new versions roughly every two weeks, an LTS release provides a **stable, validated baseline** that production environments can depend on without the risk of frequent breaking changes.

### Why ship an LTS?

The upstream OpenTelemetry Collector moves fast — new releases every ~2 weeks, components changing stability levels, deprecations, and breaking changes. This velocity is great for innovation but creates challenges for production deployments:

- **Upgrade fatigue:** Keeping up with biweekly releases is unsustainable for teams running the collector at scale.
- **Stability risk:** Not every upstream release is equally battle-tested. Some introduce regressions that get fixed in the next release.
- **Component churn:** Components get deprecated (e.g., batchprocessor), replaced, or have their APIs changed. Teams need time to plan migrations.
- **Support burden:** Supporting arbitrary collector versions is impractical. An LTS gives a defined, tested target.

The Tulip LTS provides:

- **A curated, validated component set** — every component is reviewed for stability, active maintenance, and known issues before inclusion.
- **A predictable upgrade path** — instead of chasing every upstream release, teams upgrade LTS-to-LTS with clear migration documentation.
- **Production-ready defaults** — the default configuration includes OOM protection, retry logic.
- **Extended support window** — security patches and critical fixes backported without requiring a full version upgrade.

## Overview

This document defines the component composition and configuration changes for the Tulip LTS release, targeting May 2026. The LTS follows a **stability-first** approach: only stable/beta components with active maintainers are included.

**Version bump:** v0.145.0 → v0.150.0 (all components)

The target version v0.150.0 was selected as the latest stable upstream release at the time of this LTS cut, verified against the official otelcol-contrib distribution manifest:
- https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-contrib/manifest.yaml

---

## Component Changes

### Removed: batchprocessor

**Reason:** The batchprocessor has been formally deprecated in the OpenTelemetry Collector (PR [#15046](https://github.com/open-telemetry/opentelemetry-collector/pull/15046), April 2026). It also has a known data loss bug ([#12443](https://github.com/open-telemetry/opentelemetry-collector/issues/12443)) where data is silently dropped when a downstream exporter rejects and has no queue/retry configured.

**Replacement:** Exporter-level batching via `sending_queue` + `batch` configuration on each exporter. This is the officially recommended path and provides stronger delivery guarantees because:

- Data is durably enqueued in the exporter's persistent sending queue before acknowledgment
- Batching and queueing are consolidated within the exporter
- Data can be written to disk and recovered after a Collector restart
- No silent data loss — failed sends are retried with backoff
Comment thread
perebaj marked this conversation as resolved.

#### Migration: before and after

**Before (batchprocessor in pipeline):**

```yaml
processors:
batch:
send_batch_size: 8192
timeout: 200ms

exporters:
otlp:
endpoint: otlp.example.com:4317

service:
pipelines:
traces:
receivers: [otlp]
processors: [batch]
exporters: [otlp]
```

**After (exporter-level batching):**

```yaml
exporters:
otlp:
endpoint: otlp.example.com:4317
sending_queue:
enabled: true
queue_size: 1000
batch:
flush_timeout: 200ms
min_size: 8192
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s

service:
pipelines:
traces:
receivers: [otlp]
processors: []
exporters: [otlp]
```
Comment thread
perebaj marked this conversation as resolved.

#### References

- Deprecation PR: https://github.com/open-telemetry/opentelemetry-collector/pull/15046
- Data loss bug: https://github.com/open-telemetry/opentelemetry-collector/issues/12443
- Resolution discussion: https://github.com/open-telemetry/opentelemetry-collector/issues/15047
- Docs removal: https://github.com/open-telemetry/opentelemetry-collector/issues/13766

---

### Added: memorylimiterprocessor

**Reason:** Prevents the collector from being OOM-killed in production. Without it, memory spikes from telemetry bursts or slow downstream exporters cause unbounded memory growth, leading to pod eviction (Kubernetes) or host destabilization.

This is a core component (not contrib), stable maturity, and recommended by every major OTel production deployment guide.

**Configuration example:**

```yaml
processors:
memory_limiter:
check_interval: 1s
limit_mib: 1500
spike_limit_mib: 512

service:
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter]
exporters: [otlp]
```

**Note:** `memory_limiter` should be the first processor in every pipeline to ensure backpressure is applied before any processing work is done.

---

## Final LTS Component Manifest (25 components)

### Extensions (7)

| Component | Source | Stability |
|-----------|--------|-----------|
| zpagesextension | core | stable |
| healthcheckextension | contrib | beta |
| pprofextension | contrib | beta |
| basicauthextension | contrib | beta |
| bearertokenauthextension | contrib | beta |
| oauth2clientauthextension | contrib | beta |
| oidcauthextension | contrib | beta |

### Receivers (2)

| Component | Source | Stability |
|-----------|--------|-----------|
| otlpreceiver | core | stable |
| nopreceiver | core | stable |

### Exporters (5)

| Component | Source | Stability |
|-----------|--------|-----------|
| debugexporter | core | stable |
| nopexporter | core | stable |
| otlpexporter | core | stable |
| otlphttpexporter | core | stable |
| fileexporter | contrib | beta |

### Processors (7)

| Component | Source | Stability | Note |
|-----------|--------|-----------|------|
| memorylimiterprocessor | core | stable | NEW |
| attributesprocessor | contrib | stable | |
| resourceprocessor | contrib | stable | |
| spanprocessor | contrib | stable | |
| probabilisticsamplerprocessor | contrib | stable | |
| filterprocessor | contrib | stable | |
| transformprocessor | contrib | stable | |

### Connectors (1)

| Component | Source | Stability |
|-----------|--------|-----------|
| forwardconnector | core | stable |

### Providers (3)

| Component | Source | Stability |
|-----------|--------|-----------|
| envprovider | core | stable |
| fileprovider | core | stable |
| yamlprovider | core | stable |

2 changes: 1 addition & 1 deletion test/generate-trace.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ if [[ -z $distribution ]]; then
exit 1
fi

tracegen -otlp-endpoint localhost:4317 -otlp-insecure -service e2e-test &>> ./test/logs/tracegen-${distribution}.log
tracegen -otlp-endpoint localhost:4317 -otlp-insecure -service e2e-test >> ./test/logs/tracegen-${distribution}.log 2>&1
if [ $? != 0 ]; then
echo "Failed to generate a trace."
exit 1
Expand Down