diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d8319e2..4110976d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,3 +28,16 @@ If a test case fails (lets say `examples/nodejs`), follows these steps: You can run all everything together using `mise run test`. [oats]: https://github.com/grafana/oats + +## Architecture diagram + +The source code for the architecture diagram is a [google slide](https://docs.google.com/presentation/d/1txMBBitezscvtJIXRHNSXnCekjMRM29GmHufUSI0NRw/edit?slide=id.g26040f0db78_0_0#slide=id.g26040f0db78_0_0). +Take a screenshot of the slide and save it as `img/overview.png`. + +## OTel Collector + +### Testing the combined configuration + +```shell +./otelcol-contrib --config docker/otelcol-config.yaml --config docker/otelcol-export-http.yaml print-initial-config --feature-gates otelcol.printInitialConfig > merged.yaml +``` diff --git a/docker/Dockerfile b/docker/Dockerfile index 73ffd842..d6af86d3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,73 +13,7 @@ ARG OPENTELEMETRY_COLLECTOR_VERSION=v0.123.1 FROM redhat/ubi9:9.5-1744101466 AS builder RUN mkdir /otel-lgtm -WORKDIR /otel-lgtm -ARG GRAFANA_VERSION -ARG PROMETHEUS_VERSION -ARG TEMPO_VERSION -ARG LOKI_VERSION -ARG OPENTELEMETRY_COLLECTOR_VERSION - -# TARGETARCH is automatically detected and set by the Docker daemon during the build process. If the build starts -# on an amd64 architecture, than the TARGETARCH will be set to `amd64`. -# More details on the variables can be found here: https://docs.docker.com/desktop/extensions-sdk/extensions/multi-arch/ -ARG TARGETARCH -ENV TARGETARCH=${TARGETARCH} - -# renovate: datasource=github-releases depName=cosign packageName=sigstore/cosign -ARG COSIGN_VERSION=v2.5.0 - -COPY download-grafana.sh install-cosign.sh /tmp/ - -RUN /tmp/install-cosign.sh - -# hadolint ignore=DL3033 -RUN yum install -y unzip dos2unix jq procps - -RUN /tmp/download-grafana.sh - -RUN bash -c 'ARCHIVE=prometheus-"${PROMETHEUS_VERSION:1}".linux-"${TARGETARCH}" && \ - curl -sOL https://github.com/prometheus/prometheus/releases/download/"${PROMETHEUS_VERSION}"/sha256sums.txt && \ - curl -sOL https://github.com/prometheus/prometheus/releases/download/"${PROMETHEUS_VERSION}"/"${ARCHIVE}".tar.gz && \ - sha256sum -c sha256sums.txt --ignore-missing && \ - tar xfz "${ARCHIVE}".tar.gz && \ - mv "${ARCHIVE}" prometheus && \ - rm "${ARCHIVE}".tar.gz' - -RUN bash -c 'ARCHIVE=tempo_"${TEMPO_VERSION:1}"_linux_"${TARGETARCH}".tar.gz && \ - curl -sOL https://github.com/grafana/tempo/releases/download/"${TEMPO_VERSION}"/SHA256SUMS && \ - curl -sOL https://github.com/grafana/tempo/releases/download/"${TEMPO_VERSION}"/"${ARCHIVE}" && \ - sha256sum -c SHA256SUMS --ignore-missing && \ - mkdir tempo && \ - tar xfz "${ARCHIVE}" -C tempo/ && \ - rm "${ARCHIVE}"' - -RUN bash -c 'ARCHIVE=loki-linux-"${TARGETARCH}".zip && \ - curl -sOL https://github.com/grafana/loki/releases/download/"${LOKI_VERSION}"/SHA256SUMS && \ - curl -sOL https://github.com/grafana/loki/releases/download/"${LOKI_VERSION}"/"${ARCHIVE}" && \ - sha256sum -c SHA256SUMS --ignore-missing && \ - mkdir loki && \ - unzip "${ARCHIVE}" -d loki/ && \ - rm "${ARCHIVE}" && \ - mv loki/loki-linux-"${TARGETARCH}" loki/loki' - -RUN bash -c 'ARCHIVE=otelcol-contrib_"${OPENTELEMETRY_COLLECTOR_VERSION:1}"_linux_"${TARGETARCH}".tar.gz && \ - curl -sOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/"${OPENTELEMETRY_COLLECTOR_VERSION}"/"${ARCHIVE}".sig && \ - curl -sOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/"${OPENTELEMETRY_COLLECTOR_VERSION}"/"${ARCHIVE}".pem && \ - curl -sOL https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/"${OPENTELEMETRY_COLLECTOR_VERSION}"/"${ARCHIVE}" && \ - cosign verify-blob \ - --certificate-identity-regexp github.com/open-telemetry/opentelemetry-collector-releases \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com \ - --certificate "${ARCHIVE}".pem \ - --signature "${ARCHIVE}".sig \ - "${ARCHIVE}" && \ - mkdir otelcol-contrib && \ - tar xfz "${ARCHIVE}" -C otelcol-contrib/ && \ - rm "${ARCHIVE}"' - -COPY grafana-datasources.yaml ./grafana/conf/provisioning/datasources/ -COPY grafana-dashboards.yaml ./grafana/conf/provisioning/dashboards/ COPY prometheus.yaml \ run-prometheus.sh \ grafana-dashboard-red-metrics-classic.json \ @@ -93,35 +27,64 @@ COPY prometheus.yaml \ run-tempo.sh \ otelcol-config*.yaml \ run-otelcol.sh \ - run-all.sh ./ + run-all.sh \ + /otel-lgtm/ + +# hadolint ignore=DL3033 +RUN yum install -y unzip dos2unix jq + +# installs for the final image +# see https://github.com/thomasdarimont/keycloak/blob/main/docs/guides/server/containers.adoc#installing-additional-rpm-packages +RUN mkdir -p /mnt/rootfs +RUN dnf install --installroot /mnt/rootfs curl-minimal jq procps --releasever 9 --setopt install_weak_deps=false --nodocs -y && \ + dnf --installroot /mnt/rootfs clean all && \ + rpm --root /mnt/rootfs -e --nodeps setup + # hadolint ignore=SC2038,DL4006 -RUN find . -maxdepth 1 -type f | xargs dos2unix +RUN find /otel-lgtm/ -maxdepth 1 -type f | xargs dos2unix + +COPY download-grafana.sh \ + download-prometheus.sh \ + download-tempo.sh \ + download-loki.sh \ + download-otelcol.sh \ + install-cosign.sh \ + ./ + +# TARGETARCH is automatically detected and set by the Docker daemon during the build process. If the build starts +# on an amd64 architecture, than the TARGETARCH will be set to `amd64`. +# More details on the variables can be found here: https://docs.docker.com/desktop/extensions-sdk/extensions/multi-arch/ +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} + +# renovate: datasource=github-releases depName=cosign packageName=sigstore/cosign +ARG COSIGN_VERSION=v2.5.0 + +RUN ./install-cosign.sh $COSIGN_VERSION + +ARG GRAFANA_VERSION +ARG PROMETHEUS_VERSION +ARG TEMPO_VERSION +ARG LOKI_VERSION +ARG OPENTELEMETRY_COLLECTOR_VERSION + +RUN ./download-grafana.sh $GRAFANA_VERSION +RUN ./download-prometheus.sh $PROMETHEUS_VERSION +RUN ./download-tempo.sh $TEMPO_VERSION +RUN ./download-loki.sh $LOKI_VERSION +RUN ./download-otelcol.sh $OPENTELEMETRY_COLLECTOR_VERSION + +COPY grafana-datasources.yaml /otel-lgtm/grafana/conf/provisioning/datasources/ +COPY grafana-dashboards.yaml /otel-lgtm/grafana/conf/provisioning/dashboards/ FROM redhat/ubi9-micro:9.5-1744118077 RUN mkdir /otel-lgtm WORKDIR /otel-lgtm -COPY --from=builder --chown=0:0 /usr/bin/curl /usr/bin/jq /usr/bin/grep /usr/bin/ -# jq -COPY --from=builder --chown=0:0 /usr/lib64/libjq.so.1 \ - /usr/lib64/libonig.so.5 \ - # curl - /usr/lib64/libkeyutils.so* \ - /usr/lib64/libz.so* \ - /usr/lib64/libkrb5support.so* \ - /usr/lib64/libkrb5.so* \ - /usr/lib64/libk5crypto.so* \ - /usr/lib64/libgssapi_krb5.so* \ - /usr/lib64/libcom_err.so* \ - /usr/lib64/libssl.so* \ - /usr/lib64/libcrypto.so* \ - /usr/lib64/libcurl.so* \ - /usr/lib64/libnghttp2.so* \ - /usr/lib64/libresolv.so* \ - # grep - /usr/lib64/libpcre.so.1 \ - /usr/lib64/libsigsegv.so.2 /lib64/libsigsegv.so.2 /lib64/ +COPY --from=builder /mnt/rootfs / +# to send telemetry to an external server +COPY --from=builder /etc/pki /etc/pki COPY --from=builder /otel-lgtm /otel-lgtm diff --git a/docker/download-grafana.sh b/docker/download-grafana.sh index 41856863..c0778376 100755 --- a/docker/download-grafana.sh +++ b/docker/download-grafana.sh @@ -2,12 +2,15 @@ set -euo pipefail -# too complicated to have as inline script in dockerfile +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi -ARCHIVE=grafana-"${GRAFANA_VERSION:1}".linux-"${TARGETARCH}".tar.gz +ARCHIVE=grafana-"${VERSION:1}".linux-"${TARGETARCH}".tar.gz curl -sOL https://dl.grafana.com/oss/release/"${ARCHIVE}" -CHECKSUM_URL=https://grafana.com/api/downloads/grafana/versions/"${GRAFANA_VERSION:1}"/packages/"${TARGETARCH}"/linux +CHECKSUM_URL=https://grafana.com/api/downloads/grafana/versions/"${VERSION:1}"/packages/"${TARGETARCH}"/linux echo "$(curl -sL "${CHECKSUM_URL}" -H 'accept: application/json' | jq -r '.sha256') ${ARCHIVE}" | sha256sum -c tar xfz "${ARCHIVE}" -rm "${ARCHIVE}" -mv grafana-"${GRAFANA_VERSION}" grafana/ +mv grafana-"${VERSION}" /otel-lgtm/grafana/ diff --git a/docker/download-loki.sh b/docker/download-loki.sh new file mode 100755 index 00000000..7cd6cc51 --- /dev/null +++ b/docker/download-loki.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +ARCHIVE=loki-linux-"${TARGETARCH}".zip +curl -sOL https://github.com/grafana/loki/releases/download/"${VERSION}"/SHA256SUMS +curl -sOL https://github.com/grafana/loki/releases/download/"${VERSION}"/"${ARCHIVE}" +sha256sum -c SHA256SUMS --ignore-missing +mkdir /otel-lgtm/loki +unzip "${ARCHIVE}" -d /loki/ +mv loki/loki-linux-"${TARGETARCH}" /otel-lgtm/loki/loki diff --git a/docker/download-otelcol.sh b/docker/download-otelcol.sh new file mode 100755 index 00000000..27f52968 --- /dev/null +++ b/docker/download-otelcol.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +ARCHIVE=otelcol-contrib_"${VERSION:1}"_linux_"${TARGETARCH}".tar.gz +URL=https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/"${VERSION}"/"${ARCHIVE}" +curl -sOL "${URL}".sig +curl -sOL "${URL}".pem +curl -sOL "${URL}" +cosign verify-blob \ + --certificate-identity-regexp github.com/open-telemetry/opentelemetry-collector-releases \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate "${ARCHIVE}".pem \ + --signature "${ARCHIVE}".sig \ + "${ARCHIVE}" +mkdir /otel-lgtm/otelcol-contrib +tar xfz "${ARCHIVE}" -C /otel-lgtm/otelcol-contrib/ diff --git a/docker/download-prometheus.sh b/docker/download-prometheus.sh new file mode 100755 index 00000000..665ce2a9 --- /dev/null +++ b/docker/download-prometheus.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +ARCHIVE=prometheus-"${VERSION:1}".linux-"${TARGETARCH}" +curl -sOL https://github.com/prometheus/prometheus/releases/download/"${VERSION}"/sha256sums.txt +curl -sOL https://github.com/prometheus/prometheus/releases/download/"${VERSION}"/"${ARCHIVE}".tar.gz +sha256sum -c sha256sums.txt --ignore-missing +tar xfz "${ARCHIVE}".tar.gz +mv "${ARCHIVE}" /otel-lgtm/prometheus diff --git a/docker/download-tempo.sh b/docker/download-tempo.sh new file mode 100755 index 00000000..0dd67031 --- /dev/null +++ b/docker/download-tempo.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +ARCHIVE=tempo_"${VERSION:1}"_linux_"${TARGETARCH}".tar.gz +curl -sOL https://github.com/grafana/tempo/releases/download/"${VERSION}"/SHA256SUMS +curl -sOL https://github.com/grafana/tempo/releases/download/"${VERSION}"/"${ARCHIVE}" +sha256sum -c SHA256SUMS --ignore-missing +mkdir /otel-lgtm/tempo +tar xfz "${ARCHIVE}" -C /otel-lgtm/tempo/ diff --git a/docker/install-cosign.sh b/docker/install-cosign.sh index ed827cd2..91cbdd21 100755 --- a/docker/install-cosign.sh +++ b/docker/install-cosign.sh @@ -2,9 +2,13 @@ set -euo pipefail -# too complicated to have as inline script in dockerfile +VERSION=${1:-} +if [[ -z "${VERSION}" ]]; then + echo "Usage: $0 " + exit 1 +fi +VERSION="${VERSION:1}" -VERSION=${COSIGN_VERSION:1} ARCH=$(uname -m) if [[ "${ARCH}" == "x86_64" ]]; then ARCH="x86_64" diff --git a/docker/otelcol-config-export-http.yaml b/docker/otelcol-config-export-http.yaml index c3b41584..09485dfb 100644 --- a/docker/otelcol-config-export-http.yaml +++ b/docker/otelcol-config-export-http.yaml @@ -1,56 +1,12 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - prometheus/collector: - config: - scrape_configs: - - job_name: "opentelemetry-collector" - static_configs: - - targets: ["localhost:8888"] - -processors: - batch: - -exporters: - otlphttp/metrics: - endpoint: http://localhost:9090/api/v1/otlp - tls: - insecure: true - otlphttp/traces: - endpoint: http://localhost:4418 - tls: - insecure: true - otlphttp/logs: - endpoint: http://localhost:3100/otlp - tls: - insecure: true - otlphttp/external: - endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT} - debug/metrics: - verbosity: detailed - debug/traces: - verbosity: detailed - debug/logs: - verbosity: detailed - service: pipelines: traces: - receivers: [otlp] - processors: [batch] exporters: [otlphttp/traces, otlphttp/external] - #exporters: [otlphttp/traces,debug/traces] metrics: - receivers: [otlp, prometheus/collector] - processors: [batch] exporters: [otlphttp/metrics, otlphttp/external] - #exporters: [otlphttp/metrics,debug/metrics] logs: - receivers: [otlp] - processors: [batch] exporters: [otlphttp/logs, otlphttp/external] - #exporters: [otlphttp/logs,debug/logs] + +exporters: + otlphttp/external: + endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT} diff --git a/docker/run-grafana.sh b/docker/run-grafana.sh index 32e7c3a1..febfe725 100755 --- a/docker/run-grafana.sh +++ b/docker/run-grafana.sh @@ -11,5 +11,5 @@ export GF_PATHS_HOME=/data/grafana export GF_PATHS_DATA=/data/grafana/data export GF_PATHS_PLUGINS=/data/grafana/plugins -cd ./grafana +cd /otel-lgtm/grafana || exit 1 run_with_logging "Grafana ${GRAFANA_VERSION}" "${ENABLE_LOGS_GRAFANA:-false}" ./bin/grafana server diff --git a/docker/run-otelcol.sh b/docker/run-otelcol.sh index 657c2901..81745720 100755 --- a/docker/run-otelcol.sh +++ b/docker/run-otelcol.sh @@ -2,11 +2,11 @@ source ./logging.sh -config_file="otelcol-config.yaml" +secondary_config_file="" if [[ -v OTEL_EXPORTER_OTLP_ENDPOINT ]]; then echo "Also enabling OTLP/HTTP export to ${OTEL_EXPORTER_OTLP_ENDPOINT}" - config_file="otelcol-config-export-http.yaml" + secondary_config_file="--config=file:./otelcol-config-export-http.yaml" if [[ -v OTEL_EXPORTER_OTLP_HEADERS ]]; then echo "Adding headers from OTEL_EXPORTER_OTLP_HEADERS" @@ -28,10 +28,9 @@ if [[ -v OTEL_EXPORTER_OTLP_ENDPOINT ]]; then yaml_headers+="}" # add the contents of OTEL_EXPORTER_OTLP_HEADERS to the otelcol-config-export-http.yaml file - # after "endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}" - - sed -i "s#.*endpoint: \${env:OTEL_EXPORTER_OTLP_ENDPOINT}.*#&\n headers: ${yaml_headers}#" "$config_file" + printf '\n headers: %s' "${yaml_headers}" >>otelcol-config-export-http.yaml fi fi -run_with_logging "OpenTelemetry Collector ${OPENTELEMETRY_COLLECTOR_VERSION}" "${ENABLE_LOGS_OTELCOL:-false}" ./otelcol-contrib/otelcol-contrib --config=file:./${config_file} +run_with_logging "OpenTelemetry Collector ${OPENTELEMETRY_COLLECTOR_VERSION}" "${ENABLE_LOGS_OTELCOL:-false}" \ + ./otelcol-contrib/otelcol-contrib --config=file:./otelcol-config.yaml ${secondary_config_file} diff --git a/lychee.toml b/lychee.toml index 8631beeb..281649d7 100644 --- a/lychee.toml +++ b/lychee.toml @@ -1,2 +1,4 @@ max_retries = 6 exclude_loopback = true + +exclude = ['https://docs.google.com/.*']