From d64bb466360dfea7c4e30c34b9277925de76b1f1 Mon Sep 17 00:00:00 2001 From: Tolya Korniltsev YOLO vibecoder Date: Fri, 13 Mar 2026 02:08:17 +0000 Subject: [PATCH] feat(pyroscope): Replace Parca gRPC debuginfo upload with Pyroscope Connect API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the old Parca gRPC-based debuginfo/symbol upload implementation with the new Pyroscope Connect bidirectional streaming API. This simplifies the upload protocol from a 4-step gRPC flow to a single bidi stream. Key changes: - New Connect-based PyroscopeSymbolUploader replacing ParcaSymbolUploader - Appender interface now exposes ConnectClient()/ConnectClients() instead of the old parca gRPC Client() - receive_http proxy fans out uploads to ALL downstream Connect clients - New undocumented symbol_upload_enabled config knob in pyroscope.ebpf (disabled by default, for internal use) - Removed all parca gRPC dependencies (buf.build/gen/go/parca-dev/parca) - Bumped github.com/grafana/pyroscope/api to debuginfo-upload branch Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Remove redundant ConnectClient() from Appender interface ConnectClients() (plural) already covers the single-client case, making ConnectClient() (singular) unnecessary. Remove it from the interface and all 10 implementations. Co-Authored-By: Claude Opus 4.6 (1M context) test(pyroscope): Add integration tests for PyroscopeSymbolUploader Add 7 tests exercising the uploader against a mock Connect server: - Success flow with data integrity verification - Server declining upload (cached in retry) - Upload-in-progress reason handling - Empty buildID fallback to fileID - Large file multi-chunk streaming (6.5MB → 3 chunks) - Dedup via in-progress tracker - End-to-end worker queue processing Also fix a bug where CloseResponse() after CloseRequest() canceled the HTTP/2 stream before the server finished reading buffered chunks. Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Fix CI lint errors and update collector go.mod - Replace sync/atomic with go.uber.org/atomic (depguard) - Fix gofmt formatting in appender_mock.go, receive_http_test.go, relabel_test.go - Remove unused *httptest.Server return from startMockServer (unparam) - Run go mod tidy on collector module Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Update extension/alloyengine go.mod for pyroscope/api dep Run generate-module-dependencies to sync extension/alloyengine module with updated pyroscope/api dependency and removal of parca deps. Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Address PR review — don't put fileID hash in GnuBuildId When buildID is empty, pass it as empty GnuBuildId rather than substituting the fileID hash. The fileID already goes into OtelFileId. Update TestAttemptUpload_EmptyBuildID to verify both fields. Co-Authored-By: Claude Opus 4.6 (1M context) test(pyroscope): Add integration tests for receive_http debuginfo proxy Add 5 tests for the debuginfo upload fan-out proxy in receive_http: - Single endpoint accepts upload: verifies init metadata and chunk data - Multiple endpoints all accept: all 3 receive identical data - Multiple endpoints all decline: proxy returns false, no chunks sent - Mixed accept/decline: only accepting endpoints receive chunks - No endpoints: returns connect.CodeUnavailable Also fix proxy bug: remove CloseResponse() on downstream streams that canceled HTTP/2 streams before servers finished reading buffered chunks. Co-Authored-By: Claude Opus 4.6 (1M context) feat(pyroscope): Add symb_cache_enabled config knob to pyroscope.ebpf Add undocumented symb_cache_enabled attribute (default true) to allow disabling the local irsymcache symbolizer. When false, the eBPF component skips creating the symbol cache and passes nil to the reporter, producing unsymbolized profiles. This is useful for testing server-side symbolization via debuginfo upload. Co-Authored-By: Claude Opus 4.6 (1M context) chore(pyroscope): Bump pyroscope/api to downgrade-otel branch, revert otel SDK to v1.39.0 - Bump github.com/grafana/pyroscope/api to PR #4897 revision which downgrades otel SDK dependency from v1.40.0 to v1.39.0 - Revert otel/sdk and otel/sdk/metric to v1.39.0 across all modules (go.mod, collector/go.mod, extension/alloyengine/go.mod) to fix runtime schema URL conflict between 1.37.0 and 1.39.0 - Rename parca_uploader_test.go to pyroscope_uploader_test.go Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Add h2c transport for debuginfo upload and populate InitArguments Two fixes discovered during manual integration testing: 1. The Connect debuginfo client needs HTTP/2 for bidi streaming. For HTTPS this works via ALPN, but for plain HTTP endpoints we need h2c. Add newHTTP2Client() wrapper in pyroscope.write that creates an h2c- capable transport for the debuginfo Connect client. 2. Populate UploadJob.InitArguments with default values (CacheSize=1024, QueueSize=64, WorkerNum=4) in reportExecutableForDebugInfoUpload. Without these, the LRU cache fails with "capacity must be positive". Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Skip vdso upload in debuginfo uploader Virtual DSOs (linux-vdso, [vdso]) have no backing file and no build ID, so uploading them always fails. Skip them early in Upload() to avoid unnecessary error logs. Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Address PR review comments - Remove unnecessary pyroscope_uploader_stub.go (old code never had one) - Move debuginfo InitArguments defaults to ebpf component args as a debug_info block, pass debuginfo.Arguments as-is to UploadJob - Fix h2c client: only use h2c transport for plain HTTP endpoints, pass base client through for HTTPS (HTTP/2 via ALPN) - Rename ConnectClients to DebugInfoClients across all interfaces and implementations Co-Authored-By: Claude Opus 4.6 (1M context) chore(pyroscope): Bump pyroscope/api to v1.3.2 release tag Update github.com/grafana/pyroscope/api from pseudo-version to the v1.3.2 release tag across all modules. This tag includes the otel SDK downgrade to v1.39.0, avoiding the runtime schema URL conflict. Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Remove h2c client, use HTTP/2 TLS in tests Remove the h2c transport from write.go — the debuginfo Connect client now uses the same HTTP client as the push client. For HTTPS endpoints, HTTP/2 is negotiated via ALPN automatically. Refactor proxy tests to use startProxyServer() which wraps the Component's Upload handler in an httptest TLS server with HTTP/2, eliminating the need for h2c in tests entirely. Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Add h2c transport for debuginfo upload Bidi streaming (debuginfo upload) requires HTTP/2. For HTTPS this works via ALPN, but both receive_http and pyroscope serve h2c (HTTP/2 over cleartext). Add newH2CClient() that uses an h2c transport for plain HTTP endpoints and passes through the base client for HTTPS. Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Reuse base client settings in h2c transport The h2c transport now reuses the base HTTP client's DialContext (for proxy/timeout), Timeout, CheckRedirect, and Jar settings instead of creating an isolated client that ignores HTTPClientConfig. Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Clean up h2c client, document limitation Simplify newH2CClient and document that h2c is not supported by commonconfig.NewClientFromConfig or the Go standard library. Reference internal/service/cluster/cluster.go which uses the same pattern. For HTTPS endpoints the base client is returned as-is (HTTP/2 via ALPN). Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Explicitly set UploadEnabled=false in default DebugInfoArguments Co-Authored-By: Claude Opus 4.6 (1M context) refactor(pyroscope): Remove symbol_upload_enabled, use debug_info block Remove the standalone symbol_upload_enabled flag from pyroscope.ebpf args. The upload is now controlled by the upload attr in the debug_info block (DebugInfoArguments.UploadEnabled). Co-Authored-By: Claude Opus 4.6 (1M context) fix(pyroscope): Fix gofmt and stuttering function name - Fix gofmt alignment in appender_mock.go - Fix getDebugInfoDebugInfoClients -> getDebugInfoClients stutter Co-Authored-By: Claude Opus 4.6 (1M context) --- collector/go.mod | 15 +- collector/go.sum | 30 +- extension/alloyengine/go.mod | 15 +- extension/alloyengine/go.sum | 30 +- go.mod | 15 +- go.sum | 30 +- internal/component/pyroscope/appender.go | 11 +- internal/component/pyroscope/appender_mock.go | 21 +- internal/component/pyroscope/ebpf/args.go | 11 +- .../component/pyroscope/ebpf/ebpf_linux.go | 37 +- .../parca/reporter/grpc_upload_client.go | 144 ------ .../parca/reporter/parca_uploader_test.go | 61 --- ...arca_uploader.go => pyroscope_uploader.go} | 255 ++++------- .../parca/reporter/pyroscope_uploader_test.go | 432 ++++++++++++++++++ internal/component/pyroscope/enrich/enrich.go | 6 +- .../component/pyroscope/java/loop_test.go | 4 +- .../pyroscope/receive_http/debuginfo.go | 196 +++++--- .../component/pyroscope/receive_http/grpc.go | 35 -- .../pyroscope/receive_http/receive_http.go | 12 +- .../receive_http/receive_http_test.go | 310 ++++++++++++- .../component/pyroscope/relabel/relabel.go | 6 +- .../pyroscope/relabel/relabel_test.go | 4 +- .../pyroscope/write/debuginfo/common.go | 72 ++- .../pyroscope/write/debuginfo/stub.go | 4 +- .../write/debuginfo/{parca.go => upload.go} | 12 +- .../pyroscope/write/debuginfo_client.go | 103 ----- internal/component/pyroscope/write/write.go | 58 ++- 27 files changed, 1168 insertions(+), 761 deletions(-) delete mode 100644 internal/component/pyroscope/ebpf/reporter/parca/reporter/grpc_upload_client.go delete mode 100644 internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader_test.go rename internal/component/pyroscope/ebpf/reporter/parca/reporter/{parca_uploader.go => pyroscope_uploader.go} (56%) create mode 100644 internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader_test.go delete mode 100644 internal/component/pyroscope/receive_http/grpc.go rename internal/component/pyroscope/write/debuginfo/{parca.go => upload.go} (72%) delete mode 100644 internal/component/pyroscope/write/debuginfo_client.go diff --git a/collector/go.mod b/collector/go.mod index 481d5fe8f17..0a44785d435 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -99,8 +99,6 @@ require ( ) require ( - buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 // indirect - buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 // indirect cel.dev/expr v0.24.0 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.17.0 // indirect @@ -113,7 +111,7 @@ require ( cloud.google.com/go/pubsub/v2 v2.3.0 // indirect cloud.google.com/go/storage v1.57.0 // indirect cloud.google.com/go/trace v1.11.6 // indirect - connectrpc.com/connect v1.18.1 // indirect + connectrpc.com/connect v1.19.1 // indirect cyphar.com/go-pathrs v0.2.1 // indirect dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.1 // indirect @@ -488,6 +486,7 @@ require ( github.com/google/cadvisor v0.54.1 // indirect github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a // indirect github.com/google/flatbuffers v25.9.23+incompatible // indirect + github.com/google/gnostic v0.7.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-github/v62 v62.0.0 // indirect @@ -526,7 +525,7 @@ require ( github.com/grafana/loki/v3 v3.6.2 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect - github.com/grafana/pyroscope/api v1.2.0 // indirect + github.com/grafana/pyroscope/api v1.3.2 // indirect github.com/grafana/pyroscope/ebpf v0.4.11 // indirect github.com/grafana/pyroscope/lidia v0.0.2 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect @@ -534,7 +533,7 @@ require ( github.com/grafana/vmware_exporter v0.0.5-beta.0.20250218170317-73398ba08329 // indirect github.com/grafana/walqueue v0.0.0-20260122211421-92af63e5c3dd // indirect github.com/grobie/gomemcache v0.0.0-20230213081705-239240bbc445 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/consul/api v1.32.1 // indirect @@ -773,7 +772,7 @@ require ( github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus-community/elasticsearch_exporter v1.5.0 // indirect @@ -1039,8 +1038,8 @@ require ( gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/api v0.257.0 // indirect google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/collector/go.sum b/collector/go.sum index cccf32ff9cc..41facc07538 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -1,7 +1,3 @@ -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 h1:/vx/oPlucRbONzmIhz1Fu+2DvCoURXBL/ZfZNRyUlAg= -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1/go.mod h1:RxD81JbL/yeDU/D5qmGNM36Ujw66t2SdkoZQ4fKMcf4= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 h1:e2rrifzV3Scn1NLa8cM0Ixn+dLgd+vRF/a7tYWqW8Sc= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1/go.mod h1:TR9iiFuhuMGsEil3U6KZnYmvZ1W2SY6uw5HPP+4mJU4= cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -58,8 +54,8 @@ cloud.google.com/go/storage v1.57.0 h1:4g7NB7Ta7KetVbOMpCqy89C+Vg5VE8scqlSHUPm7R cloud.google.com/go/storage v1.57.0/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= -connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= +connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= +connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= @@ -1077,6 +1073,8 @@ github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a h1:cnKkC github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a/go.mod h1:0UipONgYNVYq/tP7xau4Kr5Xlv7jcb9te+sOSDjylnQ= github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.7.1 h1:t5Kc7j/8kYr8t2u11rykRrPPovlEMG4+xdc/SpekATs= +github.com/google/gnostic v0.7.1/go.mod h1:KSw6sxnxEBFM8jLPfJd46xZP+yQcfE8XkiqfZx5zR28= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -1212,8 +1210,8 @@ github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463 h1:sAqtOzQS5u github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463/go.mod h1:d+dOGiVhuNDa4MaFXHVdnUBy/CzqlcNTooR8oM1wdTU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/grafana/pyroscope/api v1.2.0 h1:SfHDZcEZ4Vbj/Jj3bTOSpm4IDB33wLA2xBYxROhiL4U= -github.com/grafana/pyroscope/api v1.2.0/go.mod h1:CCWrMnwvTB5O+VBZfT+jO2RAvgm0GxdG2//kAWuMDhA= +github.com/grafana/pyroscope/api v1.3.2 h1:+F5JNUlM4ifoweKmRXW9cA6JsCeHzDVFV2Om3smqCxI= +github.com/grafana/pyroscope/api v1.3.2/go.mod h1:IQdc2koLAWVLlWcvBV4bm6uSFW2LiklKa8xyevsZ28I= github.com/grafana/pyroscope/ebpf v0.4.11 h1:QpXj3xIWveFy1Zx7M9YzCq7z+d70LQ2pF/qEzD/tXWo= github.com/grafana/pyroscope/ebpf v0.4.11/go.mod h1:LhmNuYZpxlmjsLK36j1nD+eJ/CNebBRDPCzRJPVHZbI= github.com/grafana/pyroscope/lidia v0.0.2 h1:DsZiUjWfB+6w9wKGosEQQTFuDvkay7kUECw3Os7Xno0= @@ -1234,8 +1232,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 h1:jP1RStw811EvUDzsUQ9oESqw2e4RqCjSAD9qIL8eMns= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5/go.mod h1:WXNBZ64q3+ZUemCMXD9kYnr56H7CgZxDBHCVwstfl3s= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -2087,8 +2085,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -3250,10 +3248,10 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 h1:7LRqPCEdE4TP4/9psdaB7F2nhZFfBiGJomA5sojLWdU= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/extension/alloyengine/go.mod b/extension/alloyengine/go.mod index b4e7837cc64..a55a698dcb1 100644 --- a/extension/alloyengine/go.mod +++ b/extension/alloyengine/go.mod @@ -17,8 +17,6 @@ require ( ) require ( - buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 // indirect - buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 // indirect cel.dev/expr v0.24.0 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.17.0 // indirect @@ -31,7 +29,7 @@ require ( cloud.google.com/go/pubsub/v2 v2.3.0 // indirect cloud.google.com/go/storage v1.57.0 // indirect cloud.google.com/go/trace v1.11.6 // indirect - connectrpc.com/connect v1.18.1 // indirect + connectrpc.com/connect v1.19.1 // indirect cyphar.com/go-pathrs v0.2.1 // indirect dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.1.1 // indirect @@ -405,6 +403,7 @@ require ( github.com/google/cadvisor v0.54.1 // indirect github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a // indirect github.com/google/flatbuffers v25.9.23+incompatible // indirect + github.com/google/gnostic v0.7.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-github/v62 v62.0.0 // indirect @@ -443,7 +442,7 @@ require ( github.com/grafana/loki/v3 v3.6.2 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect - github.com/grafana/pyroscope/api v1.2.0 // indirect + github.com/grafana/pyroscope/api v1.3.2 // indirect github.com/grafana/pyroscope/ebpf v0.4.11 // indirect github.com/grafana/pyroscope/lidia v0.0.2 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect @@ -451,7 +450,7 @@ require ( github.com/grafana/vmware_exporter v0.0.5-beta.0.20250218170317-73398ba08329 // indirect github.com/grafana/walqueue v0.0.0-20260122211421-92af63e5c3dd // indirect github.com/grobie/gomemcache v0.0.0-20230213081705-239240bbc445 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/consul/api v1.32.1 // indirect @@ -732,7 +731,7 @@ require ( github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus-community/elasticsearch_exporter v1.5.0 // indirect @@ -1007,8 +1006,8 @@ require ( gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/api v0.257.0 // indirect google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect diff --git a/extension/alloyengine/go.sum b/extension/alloyengine/go.sum index ea9a8e5a923..f37874ceff4 100644 --- a/extension/alloyengine/go.sum +++ b/extension/alloyengine/go.sum @@ -1,7 +1,3 @@ -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 h1:/vx/oPlucRbONzmIhz1Fu+2DvCoURXBL/ZfZNRyUlAg= -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1/go.mod h1:RxD81JbL/yeDU/D5qmGNM36Ujw66t2SdkoZQ4fKMcf4= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 h1:e2rrifzV3Scn1NLa8cM0Ixn+dLgd+vRF/a7tYWqW8Sc= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1/go.mod h1:TR9iiFuhuMGsEil3U6KZnYmvZ1W2SY6uw5HPP+4mJU4= cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -58,8 +54,8 @@ cloud.google.com/go/storage v1.57.0 h1:4g7NB7Ta7KetVbOMpCqy89C+Vg5VE8scqlSHUPm7R cloud.google.com/go/storage v1.57.0/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= -connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= +connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= +connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= @@ -1094,6 +1090,8 @@ github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a h1:cnKkC github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a/go.mod h1:0UipONgYNVYq/tP7xau4Kr5Xlv7jcb9te+sOSDjylnQ= github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.7.1 h1:t5Kc7j/8kYr8t2u11rykRrPPovlEMG4+xdc/SpekATs= +github.com/google/gnostic v0.7.1/go.mod h1:KSw6sxnxEBFM8jLPfJd46xZP+yQcfE8XkiqfZx5zR28= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -1230,8 +1228,8 @@ github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463 h1:sAqtOzQS5u github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463/go.mod h1:d+dOGiVhuNDa4MaFXHVdnUBy/CzqlcNTooR8oM1wdTU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/grafana/pyroscope/api v1.2.0 h1:SfHDZcEZ4Vbj/Jj3bTOSpm4IDB33wLA2xBYxROhiL4U= -github.com/grafana/pyroscope/api v1.2.0/go.mod h1:CCWrMnwvTB5O+VBZfT+jO2RAvgm0GxdG2//kAWuMDhA= +github.com/grafana/pyroscope/api v1.3.2 h1:+F5JNUlM4ifoweKmRXW9cA6JsCeHzDVFV2Om3smqCxI= +github.com/grafana/pyroscope/api v1.3.2/go.mod h1:IQdc2koLAWVLlWcvBV4bm6uSFW2LiklKa8xyevsZ28I= github.com/grafana/pyroscope/ebpf v0.4.11 h1:QpXj3xIWveFy1Zx7M9YzCq7z+d70LQ2pF/qEzD/tXWo= github.com/grafana/pyroscope/ebpf v0.4.11/go.mod h1:LhmNuYZpxlmjsLK36j1nD+eJ/CNebBRDPCzRJPVHZbI= github.com/grafana/pyroscope/lidia v0.0.2 h1:DsZiUjWfB+6w9wKGosEQQTFuDvkay7kUECw3Os7Xno0= @@ -1252,8 +1250,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 h1:jP1RStw811EvUDzsUQ9oESqw2e4RqCjSAD9qIL8eMns= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5/go.mod h1:WXNBZ64q3+ZUemCMXD9kYnr56H7CgZxDBHCVwstfl3s= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -2079,8 +2077,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -3247,10 +3245,10 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 h1:7LRqPCEdE4TP4/9psdaB7F2nhZFfBiGJomA5sojLWdU= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= diff --git a/go.mod b/go.mod index 601feec736f..2c18b23843c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace github.com/grafana/alloy/syntax => ./syntax require ( cloud.google.com/go/pubsub/v2 v2.3.0 - connectrpc.com/connect v1.18.1 + connectrpc.com/connect v1.19.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/go-autorest/autorest v0.11.30 @@ -78,7 +78,7 @@ require ( github.com/grafana/loki/pkg/push v0.0.0-20251125172520-2f85998f1adf github.com/grafana/loki/v3 v3.6.2 github.com/grafana/pyroscope-go/godeltaprof v0.1.8 - github.com/grafana/pyroscope/api v1.2.0 + github.com/grafana/pyroscope/api v1.3.2 github.com/grafana/pyroscope/ebpf v0.4.11 github.com/grafana/pyroscope/lidia v0.0.2 github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 @@ -649,7 +649,7 @@ require ( github.com/grafana/jvmtools v0.0.3 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect github.com/grobie/gomemcache v0.0.0-20230213081705-239240bbc445 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/cronexpr v1.1.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -830,7 +830,7 @@ require ( github.com/pires/go-proxyproto v0.7.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus-community/go-runit v0.1.0 // indirect github.com/prometheus-community/prom-label-proxy v0.12.1 // indirect @@ -978,8 +978,8 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -1006,8 +1006,6 @@ require ( ) require ( - buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 - buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 github.com/spf13/viper v1.19.0 github.com/zricethezav/gitleaks/v8 v8.30.0 ) @@ -1040,6 +1038,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/google/gnostic v0.7.1 // indirect github.com/google/go-github/v62 v62.0.0 // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/leodido/go-urn v1.4.0 // indirect diff --git a/go.sum b/go.sum index 4c5054c87e1..513f1fd1e36 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1 h1:/vx/oPlucRbONzmIhz1Fu+2DvCoURXBL/ZfZNRyUlAg= -buf.build/gen/go/parca-dev/parca/grpc/go v1.6.0-20251203114737-dab2f094ec25.1/go.mod h1:RxD81JbL/yeDU/D5qmGNM36Ujw66t2SdkoZQ4fKMcf4= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1 h1:e2rrifzV3Scn1NLa8cM0Ixn+dLgd+vRF/a7tYWqW8Sc= -buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20251203114737-dab2f094ec25.1/go.mod h1:TR9iiFuhuMGsEil3U6KZnYmvZ1W2SY6uw5HPP+4mJU4= cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -58,8 +54,8 @@ cloud.google.com/go/storage v1.57.0 h1:4g7NB7Ta7KetVbOMpCqy89C+Vg5VE8scqlSHUPm7R cloud.google.com/go/storage v1.57.0/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= -connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= +connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= +connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= @@ -1104,6 +1100,8 @@ github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a h1:cnKkC github.com/google/dnsmasq_exporter v0.2.1-0.20230620100026-44b14480804a/go.mod h1:0UipONgYNVYq/tP7xau4Kr5Xlv7jcb9te+sOSDjylnQ= github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.7.1 h1:t5Kc7j/8kYr8t2u11rykRrPPovlEMG4+xdc/SpekATs= +github.com/google/gnostic v0.7.1/go.mod h1:KSw6sxnxEBFM8jLPfJd46xZP+yQcfE8XkiqfZx5zR28= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -1240,8 +1238,8 @@ github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463 h1:sAqtOzQS5u github.com/grafana/prometheus v1.8.2-0.20260302171028-8cf60eef5463/go.mod h1:d+dOGiVhuNDa4MaFXHVdnUBy/CzqlcNTooR8oM1wdTU= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/grafana/pyroscope/api v1.2.0 h1:SfHDZcEZ4Vbj/Jj3bTOSpm4IDB33wLA2xBYxROhiL4U= -github.com/grafana/pyroscope/api v1.2.0/go.mod h1:CCWrMnwvTB5O+VBZfT+jO2RAvgm0GxdG2//kAWuMDhA= +github.com/grafana/pyroscope/api v1.3.2 h1:+F5JNUlM4ifoweKmRXW9cA6JsCeHzDVFV2Om3smqCxI= +github.com/grafana/pyroscope/api v1.3.2/go.mod h1:IQdc2koLAWVLlWcvBV4bm6uSFW2LiklKa8xyevsZ28I= github.com/grafana/pyroscope/ebpf v0.4.11 h1:QpXj3xIWveFy1Zx7M9YzCq7z+d70LQ2pF/qEzD/tXWo= github.com/grafana/pyroscope/ebpf v0.4.11/go.mod h1:LhmNuYZpxlmjsLK36j1nD+eJ/CNebBRDPCzRJPVHZbI= github.com/grafana/pyroscope/lidia v0.0.2 h1:DsZiUjWfB+6w9wKGosEQQTFuDvkay7kUECw3Os7Xno0= @@ -1262,8 +1260,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5 h1:jP1RStw811EvUDzsUQ9oESqw2e4RqCjSAD9qIL8eMns= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.5/go.mod h1:WXNBZ64q3+ZUemCMXD9kYnr56H7CgZxDBHCVwstfl3s= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -2094,8 +2092,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -3270,10 +3268,10 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2 h1:7LRqPCEdE4TP4/9psdaB7F2nhZFfBiGJomA5sojLWdU= -google.golang.org/genproto/googleapis/api v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516 h1:vmC/ws+pLzWjj/gzApyoZuSVrDtF1aod4u/+bbj8hgM= +google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 h1:sNrWoksmOyF5bvJUcnmbeAmQi8baNhqg5IWaI3llQqU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= diff --git a/internal/component/pyroscope/appender.go b/internal/component/pyroscope/appender.go index d8c8f55f003..62c75dc7a55 100644 --- a/internal/component/pyroscope/appender.go +++ b/internal/component/pyroscope/appender.go @@ -6,8 +6,8 @@ import ( "sync" "time" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/hashicorp/go-multierror" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" @@ -61,15 +61,14 @@ type Fanout struct { writeLatency prometheus.Histogram } -func (f *Fanout) Client() debuginfogrpc.DebuginfoServiceClient { +func (f *Fanout) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { f.mut.RLock() defer f.mut.RUnlock() + var clients []debuginfov1alpha1connect.DebuginfoServiceClient for _, c := range f.children { - if client := c.Client(); client != nil { - return client - } + clients = append(clients, c.DebugInfoClients()...) } - return nil + return clients } func (f *Fanout) Upload(j debuginfo.UploadJob) { diff --git a/internal/component/pyroscope/appender_mock.go b/internal/component/pyroscope/appender_mock.go index 458a75112cd..93e67e19aa5 100644 --- a/internal/component/pyroscope/appender_mock.go +++ b/internal/component/pyroscope/appender_mock.go @@ -3,18 +3,18 @@ package pyroscope import ( "context" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/prometheus/prometheus/model/labels" ) var _ Appendable = AppenderMock{} type AppenderMock struct { - AppendIngestFunc func(ctx context.Context, profile *IncomingProfile) error - AppendFunc func(ctx context.Context, labels labels.Labels, samples []*RawSample) error - ClientFunc func() debuginfogrpc.DebuginfoServiceClient - DebugInfoUploadFunc func(j debuginfo.UploadJob) + AppendIngestFunc func(ctx context.Context, profile *IncomingProfile) error + AppendFunc func(ctx context.Context, labels labels.Labels, samples []*RawSample) error + DebugInfoClientsFunc func() []debuginfov1alpha1connect.DebuginfoServiceClient + DebugInfoUploadFunc func(j debuginfo.UploadJob) } func (a AppenderMock) Append(ctx context.Context, labels labels.Labels, samples []*RawSample) error { @@ -29,12 +29,17 @@ func (a AppenderMock) Appender() Appender { return a } -func (a AppenderMock) Client() debuginfogrpc.DebuginfoServiceClient { - return a.ClientFunc() +func (a AppenderMock) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + if a.DebugInfoClientsFunc != nil { + return a.DebugInfoClientsFunc() + } + return nil } func (a AppenderMock) Upload(j debuginfo.UploadJob) { - a.DebugInfoUploadFunc(j) + if a.DebugInfoUploadFunc != nil { + a.DebugInfoUploadFunc(j) + } } func AppendableFunc(f func(ctx context.Context, labels labels.Labels, samples []*RawSample) error) AppenderMock { diff --git a/internal/component/pyroscope/ebpf/args.go b/internal/component/pyroscope/ebpf/args.go index 2ca0ee4b465..6de62c0ebe6 100644 --- a/internal/component/pyroscope/ebpf/args.go +++ b/internal/component/pyroscope/ebpf/args.go @@ -5,6 +5,7 @@ import ( "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" ) type Arguments struct { @@ -29,10 +30,12 @@ type Arguments struct { DeprecatedArguments DeprecatedArguments `alloy:",squash"` // undocumented - PyroscopeDynamicProfilingPolicy bool `alloy:"targets_only,attr,optional"` - SymbCachePath string `alloy:"symb_cache_path,attr,optional"` - SymbCacheSizeEntries int `alloy:"symb_cache_size,attr,optional"` - ReporterUnsymbolizedStubs bool `alloy:"reporter_unsymbolized_stubs,attr,optional"` + PyroscopeDynamicProfilingPolicy bool `alloy:"targets_only,attr,optional"` + SymbCachePath string `alloy:"symb_cache_path,attr,optional"` + SymbCacheSizeEntries int `alloy:"symb_cache_size,attr,optional"` + ReporterUnsymbolizedStubs bool `alloy:"reporter_unsymbolized_stubs,attr,optional"` + SymbCacheEnabled bool `alloy:"symb_cache_enabled,attr,optional"` + DebugInfoArguments debuginfo.Arguments `alloy:"debug_info,block,optional"` } type DeprecatedArguments struct { diff --git a/internal/component/pyroscope/ebpf/ebpf_linux.go b/internal/component/pyroscope/ebpf/ebpf_linux.go index 1b367e7f2b0..f0bc716715f 100644 --- a/internal/component/pyroscope/ebpf/ebpf_linux.go +++ b/internal/component/pyroscope/ebpf/ebpf_linux.go @@ -67,17 +67,20 @@ func New(logger log.Logger, reg prometheus.Registerer, id string, args Arguments appendable := pyroscope.NewFanout(args.ForwardTo, id, reg) - nfs, err := irsymcache.NewFSCache(logger, irsymcache.TableTableFactory{ - Options: []lidia.Option{ - lidia.WithFiles(), - lidia.WithLines(), - }, - }, irsymcache.Options{ - SizeEntries: uint32(args.SymbCacheSizeEntries), - Path: args.SymbCachePath, - }) - if err != nil { - return nil, err + var nfs *irsymcache.Resolver + if args.SymbCacheEnabled { + nfs, err = irsymcache.NewFSCache(logger, irsymcache.TableTableFactory{ + Options: []lidia.Option{ + lidia.WithFiles(), + lidia.WithLines(), + }, + }, irsymcache.Options{ + SizeEntries: uint32(args.SymbCacheSizeEntries), + Path: args.SymbCachePath, + }) + if err != nil { + return nil, err + } } if dynamicProfilingPolicy { @@ -279,9 +282,11 @@ func (c *Component) ReportExecutable(md *reporter2.ExecutableMetadata) { if c.symbols != nil { c.symbols.ReportExecutable(md) } + if c.args.DebugInfoArguments.UploadEnabled { + c.reportExecutableForDebugInfoUpload(md) + } } -//nolint:unused func (c *Component) reportExecutableForDebugInfoUpload(args *reporter2.ExecutableMetadata) { extractAsFile := func(pid libpf.PID, file string) string { return path.Join("/proc", strconv.Itoa(int(pid)), "root", file) @@ -304,6 +309,7 @@ func (c *Component) reportExecutableForDebugInfoUpload(args *reporter2.Executabl c.appendable.Upload(debuginfo.UploadJob{ FrameMappingFileData: mf, Open: open, + InitArguments: c.args.DebugInfoArguments, }) } @@ -331,6 +337,13 @@ func NewDefaultArguments() Arguments { PyroscopeDynamicProfilingPolicy: true, SymbCachePath: "/tmp/symb-cache", SymbCacheSizeEntries: 2048, + SymbCacheEnabled: true, + DebugInfoArguments: debuginfo.Arguments{ + UploadEnabled: false, + CacheSize: 1024, + QueueSize: 64, + WorkerNum: 4, + }, } } diff --git a/internal/component/pyroscope/ebpf/reporter/parca/reporter/grpc_upload_client.go b/internal/component/pyroscope/ebpf/reporter/parca/reporter/grpc_upload_client.go deleted file mode 100644 index 06b2874b0a1..00000000000 --- a/internal/component/pyroscope/ebpf/reporter/parca/reporter/grpc_upload_client.go +++ /dev/null @@ -1,144 +0,0 @@ -//go:build linux && (arm64 || amd64) - -// Copyright 2022-2024 The Parca Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reporter - -import ( - "bufio" - "context" - "errors" - "fmt" - "io" - - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" - debuginfopb "buf.build/gen/go/parca-dev/parca/protocolbuffers/go/parca/debuginfo/v1alpha1" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var ErrDebuginfoAlreadyExists = errors.New("debug info already exists") - -const ( - ChunkSize = 1024 * 1024 * 3 - MaxMsgSize = 1024 * 1024 * 4 -) - -type GrpcDebuginfoUploadServiceClient interface { - Upload(ctx context.Context, opts ...grpc.CallOption) (debuginfogrpc.DebuginfoService_UploadClient, error) -} - -type GrpcUploadClient struct { - GrpcDebuginfoUploadServiceClient -} - -func NewGrpcUploadClient(client GrpcDebuginfoUploadServiceClient) *GrpcUploadClient { - return &GrpcUploadClient{client} -} - -func (c *GrpcUploadClient) Upload(ctx context.Context, uploadInstructions *debuginfopb.UploadInstructions, r io.Reader) (uint64, error) { - return c.grpcUpload(ctx, uploadInstructions, r) -} - -func (c *GrpcUploadClient) grpcUpload(ctx context.Context, uploadInstructions *debuginfopb.UploadInstructions, r io.Reader) (uint64, error) { - stream, err := c.GrpcDebuginfoUploadServiceClient.Upload(ctx, grpc.MaxCallSendMsgSize(MaxMsgSize)) - if err != nil { - return 0, fmt.Errorf("initiate upload: %w", err) - } - - defer func() { - if stream != nil { - _, _ = stream.CloseAndRecv() - } - }() - - err = stream.Send(&debuginfopb.UploadRequest{ - Data: &debuginfopb.UploadRequest_Info{ - Info: &debuginfopb.UploadInfo{ - UploadId: uploadInstructions.UploadId, - BuildId: uploadInstructions.BuildId, - Type: uploadInstructions.Type, - }, - }, - }) - if err != nil { - if err := sentinelError(err); err != nil { - return 0, err - } - return 0, fmt.Errorf("send upload info: %w", err) - } - - reader := bufio.NewReader(r) - - buffer := make([]byte, ChunkSize) - - bytesSent := 0 - for { - n, err := reader.Read(buffer) - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return 0, fmt.Errorf("read next chunk (%d bytes sent so far): %w", bytesSent, err) - } - - err = stream.Send(&debuginfopb.UploadRequest{ - Data: &debuginfopb.UploadRequest_ChunkData{ - ChunkData: buffer[:n], - }, - }) - bytesSent += n - if errors.Is(err, io.EOF) { - // When the stream is closed, the server will send an EOF. - // To get the correct error code, we need the status. - // So receive the message and check the status. - err = stream.RecvMsg(nil) - if err := sentinelError(err); err != nil { - return 0, err - } - return 0, fmt.Errorf("send chunk: %w", err) - } - if err != nil { - return 0, fmt.Errorf("send next chunk (%d bytes sent so far): %w", bytesSent, err) - } - } - - // It returns io.EOF when the stream completes successfully. - res, err := stream.CloseAndRecv() - if errors.Is(err, io.EOF) { - return res.Size, nil - } - if err != nil { - // On any other error, the stream is aborted and the error contains the RPC status. - if err := sentinelError(err); err != nil { - return 0, err - } - return 0, fmt.Errorf("close and receive: %w", err) - } - return res.Size, nil -} - -// sentinelError checks underlying error for grpc.StatusCode and returns if it's a known and expected error. -func sentinelError(err error) error { - if sts, ok := status.FromError(err); ok { - if sts.Code() == codes.AlreadyExists { - return ErrDebuginfoAlreadyExists - } - if sts.Code() == codes.FailedPrecondition { - return err - } - } - return nil -} diff --git a/internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader_test.go b/internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader_test.go deleted file mode 100644 index 860f4ecdbb6..00000000000 --- a/internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader_test.go +++ /dev/null @@ -1,61 +0,0 @@ -//go:build linux && (arm64 || amd64) - -package reporter - -import ( - "math/rand" - "testing" - - "go.opentelemetry.io/ebpf-profiler/libpf" -) - -func TestMapShrink(t *testing.T) { - tr := newInProgressTracker(0.2) - r := rand.New(rand.NewSource(0)) - - items := make([]libpf.FileID, 100) - for i := 0; i < 100; i++ { - items[i] = libpf.NewFileID( - r.Uint64(), - r.Uint64(), - ) - - tr.GetOrAdd(items[i]) - } - - if tr.maxSizeSeen != 100 { - t.Errorf("expected 100, got %d", tr.maxSizeSeen) - } - - for i := 0; i < 10; i++ { - tr.Remove(items[i]) - } - - if tr.maxSizeSeen != 100 { - t.Errorf("expected 100, got %d", tr.maxSizeSeen) - } - - for i := 10; i < 20; i++ { - tr.Remove(items[i]) - } - - if tr.maxSizeSeen != 83 { - t.Errorf("expected 83, got %d", tr.maxSizeSeen) - } - - // adding up to 83 doesn't change anything - for i := 10; i < 13; i++ { - tr.GetOrAdd(items[i]) - } - - if tr.maxSizeSeen != 83 { - t.Errorf("expected 83, got %d", tr.maxSizeSeen) - } - - // adding 84th item should increases the max size - tr.GetOrAdd(items[13]) - - if tr.maxSizeSeen != 84 { - t.Errorf("expected 84, got %d", tr.maxSizeSeen) - } -} diff --git a/internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader.go b/internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader.go similarity index 56% rename from internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader.go rename to internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader.go index 86bbba7ce11..3953fb6aadd 100644 --- a/internal/component/pyroscope/ebpf/reporter/parca/reporter/parca_uploader.go +++ b/internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader.go @@ -8,16 +8,16 @@ import ( "fmt" "io" "maps" - "net/http" "os" "path/filepath" + "strings" "sync" "time" "github.com/grafana/alloy/internal/component/pyroscope/ebpf/reporter/parca/reporter/elfwriter" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" - debuginfopb "buf.build/gen/go/parca-dev/parca/protocolbuffers/go/parca/debuginfo/v1alpha1" + debuginfov1alpha1 "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -25,26 +25,30 @@ import ( lru "github.com/elastic/go-freelru" "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" //nolint:depguard - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/process" ) +const ( + ChunkSize = 1024 * 1024 * 3 +) + +const ( + ReasonUploadInProgress = "A previous upload is still in-progress and not stale yet (only stale uploads can be retried)." +) + type uploadRequest struct { fileID libpf.FileID fileName string buildID string open func() (process.ReadAtCloser, error) - client debuginfogrpc.DebuginfoServiceClient + client debuginfov1alpha1connect.DebuginfoServiceClient } -type ParcaSymbolUploader struct { +type PyroscopeSymbolUploader struct { logger log.Logger - httpClient *http.Client - retry *lru.SyncedLRU[libpf.FileID, struct{}] stripTextSection bool @@ -57,7 +61,7 @@ type ParcaSymbolUploader struct { uploadRequestBytes prometheus.Counter } -func NewParcaSymbolUploader( +func NewPyroscopeSymbolUploader( logger log.Logger, cacheSize uint32, stripTextSection bool, @@ -65,7 +69,7 @@ func NewParcaSymbolUploader( workerNum int, cacheDir string, uploadRequestBytes prometheus.Counter, -) (*ParcaSymbolUploader, error) { +) (*PyroscopeSymbolUploader, error) { retryCache, err := lru.NewSynced[libpf.FileID, struct{}](cacheSize, libpf.FileID.Hash32) if err != nil { @@ -103,9 +107,8 @@ func NewParcaSymbolUploader( return nil, fmt.Errorf("failed to clean cache directory (%s): %s", cacheDirectory, err) } - return &ParcaSymbolUploader{ + return &PyroscopeSymbolUploader{ logger: logger, - httpClient: http.DefaultClient, retry: retryCache, stripTextSection: stripTextSection, tmp: cacheDirectory, @@ -116,10 +119,6 @@ func NewParcaSymbolUploader( }, nil } -const ( - ReasonUploadInProgress = "A previous upload is still in-progress and not stale yet (only stale uploads can be retried)." -) - // inProgressTracker is a simple in-progress tracker that keeps track of which // fileIDs are currently in-progress/enqueued to be uploaded. type inProgressTracker struct { @@ -173,8 +172,8 @@ func (i *inProgressTracker) Remove(fileID libpf.FileID) { } } -// Start starts the upload workers. -func (u *ParcaSymbolUploader) Run(ctx context.Context) error { +// Run starts the upload workers. +func (u *PyroscopeSymbolUploader) Run(ctx context.Context) error { var g errgroup.Group for i := 0; i < u.workerNum; i++ { @@ -197,10 +196,15 @@ func (u *ParcaSymbolUploader) Run(ctx context.Context) error { // Upload enqueues a file for upload if it's not already in progress, or if it // is marked not to be retried. -func (u *ParcaSymbolUploader) Upload(ctx context.Context, client debuginfogrpc.DebuginfoServiceClient, +func (u *PyroscopeSymbolUploader) Upload(ctx context.Context, client debuginfov1alpha1connect.DebuginfoServiceClient, fileID libpf.FileID, fileName string, buildID string, open func() (process.ReadAtCloser, error)) { + // Skip virtual DSOs — they have no backing file and no build ID. + if strings.HasPrefix(fileName, "linux-vdso") || strings.HasPrefix(fileName, "[vdso]") { + return + } + _, ok := u.retry.Get(fileID) if ok { return @@ -224,25 +228,48 @@ func (u *ParcaSymbolUploader) Upload(ctx context.Context, client debuginfogrpc.D } } -// attemptUpload attempts to upload the file with the given fileID and buildID. -func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginfogrpc.DebuginfoServiceClient, fileID libpf.FileID, fileName string, buildID string, +// attemptUpload attempts to upload the file with the given fileID and buildID +// using the new Connect bidirectional streaming API. +func (u *PyroscopeSymbolUploader) attemptUpload(ctx context.Context, client debuginfov1alpha1connect.DebuginfoServiceClient, + fileID libpf.FileID, fileName string, buildID string, open func() (process.ReadAtCloser, error)) error { defer u.inProgressTracker.Remove(fileID) - buildIDType := debuginfopb.BuildIDType_BUILD_ID_TYPE_GNU - if buildID == "" { - buildIDType = debuginfopb.BuildIDType_BUILD_ID_TYPE_HASH - buildID = fileID.StringNoQuotes() + fileType := debuginfov1alpha1.FileMetadata_TYPE_EXECUTABLE_FULL + if u.stripTextSection { + fileType = debuginfov1alpha1.FileMetadata_TYPE_EXECUTABLE_NO_TEXT + } + + // Open bidi stream. + stream := client.Upload(ctx) + + // Step 1: Send ShouldInitiateUploadRequest. + if err := stream.Send(&debuginfov1alpha1.UploadRequest{ + Data: &debuginfov1alpha1.UploadRequest_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadRequest{ + File: &debuginfov1alpha1.FileMetadata{ + GnuBuildId: buildID, + OtelFileId: fileID.StringNoQuotes(), + Name: fileName, + Type: fileType, + }, + }, + }, + }); err != nil { + return fmt.Errorf("send init request: %w", err) } - shouldInitiateUploadResp, err := client.ShouldInitiateUpload(ctx, &debuginfopb.ShouldInitiateUploadRequest{ - BuildId: buildID, - BuildIdType: buildIDType, - Type: debuginfopb.DebuginfoType_DEBUGINFO_TYPE_DEBUGINFO_UNSPECIFIED, - }) + // Step 2: Receive ShouldInitiateUploadResponse. + resp, err := stream.Receive() if err != nil { - return err + return fmt.Errorf("receive init response: %w", err) + } + + initResp := resp.GetInit() + if initResp == nil { + u.retry.Add(fileID, struct{}{}) + return fmt.Errorf("unexpected response type, expected init response") } l := log.With(u.logger, @@ -252,15 +279,11 @@ func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginf ) level.Debug(l).Log("msg", "ShouldInitiateUpload result", - "should_initiate_upload", shouldInitiateUploadResp.ShouldInitiateUpload, - "reason", shouldInitiateUploadResp.Reason) - - if !shouldInitiateUploadResp.ShouldInitiateUpload { - // This can happen when two agents simultaneously try to upload the - // same file. The other agent already started the upload so we don't - // need to do it again, however the upload may fail so we should retry - // after a while. - if shouldInitiateUploadResp.Reason == ReasonUploadInProgress { + "should_initiate_upload", initResp.ShouldInitiateUpload, + "reason", initResp.Reason) + + if !initResp.ShouldInitiateUpload { + if initResp.Reason == ReasonUploadInProgress { u.retry.AddWithLifetime(fileID, struct{}{}, 5*time.Minute) return nil } @@ -268,22 +291,15 @@ func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginf return nil } - var ( - r io.Reader - size int64 - ) + // Step 3: Prepare the file data. + var r io.Reader if !u.stripTextSection { - // We're not stripping the text section so we can upload the original file. f, err := open() if err != nil { if os.IsNotExist(err) { - // File doesn't exist, likely because the process is already - // gone. return nil } if err.Error() == "no backing file for anonymous memory" { - // This is an anonymous memory mapping, it's not backed by - // a file so we will never be able to extract debuginfo. u.retry.Add(fileID, struct{}{}) return nil } @@ -291,12 +307,11 @@ func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginf } defer f.Close() - size, err = readAtCloserSize(u.logger, f) + size, err := readAtCloserSize(u.logger, f) if err != nil { return err } if size == 0 { - // The original file is empty no need to ever upload it. u.retry.Add(fileID, struct{}{}) return nil } @@ -312,15 +327,10 @@ func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginf original, err := open() if err != nil { - os.Remove(f.Name()) if os.IsNotExist(err) { - // Original file doesn't exist the process is likely - // already gone. return nil } if err.Error() == "no backing file for anonymous memory" { - // This is an anonymous memory mapping, it's not backed by - // a file so we will never be able to extract debuginfo. u.retry.Add(fileID, struct{}{}) return nil } @@ -329,111 +339,61 @@ func (u *ParcaSymbolUploader) attemptUpload(ctx context.Context, client debuginf defer original.Close() if err := elfwriter.OnlyKeepDebug(f, original); err != nil { - os.Remove(f.Name()) - // If we can't extract the debuginfo we can't upload the file. u.retry.Add(fileID, struct{}{}) return fmt.Errorf("extract debuginfo: %w", err) } if _, err := f.Seek(0, io.SeekStart); err != nil { - os.Remove(f.Name()) - // Something is probably seriously wrong so don't retry. u.retry.Add(fileID, struct{}{}) return fmt.Errorf("seek extracted debuginfo to start: %w", err) } stat, err := f.Stat() if err != nil { - os.Remove(f.Name()) - // Something is probably seriously wrong so don't retry. u.retry.Add(fileID, struct{}{}) return fmt.Errorf("stat file to upload: %w", err) } - size = stat.Size() - - if size == 0 { - os.Remove(f.Name()) - // Extraction is a deterministic process so if the file is empty we - // will never be able to extract non-zero debuginfo the original - // binary. + if stat.Size() == 0 { u.retry.Add(fileID, struct{}{}) return nil } r = f } - initiateUploadResp, err := client.InitiateUpload(ctx, &debuginfopb.InitiateUploadRequest{ - BuildId: buildID, - BuildIdType: buildIDType, - Type: debuginfopb.DebuginfoType_DEBUGINFO_TYPE_DEBUGINFO_UNSPECIFIED, - Hash: fileID.StringNoQuotes(), - Size: size, - }) - - if err != nil { - level.Debug(u.logger).Log("msg", "InitiateUpload", "err", err) - if status.Code(err) == codes.FailedPrecondition { - // This is a race that can happen when multiple agents are trying - // to upload the same file. This happens when another upload is - // still in progress. Since we don't know if it will succeed or not - // we retry after a while. - u.retry.AddWithLifetime(fileID, struct{}{}, 5*time.Minute) - return nil - } - if status.Code(err) == codes.AlreadyExists { - // This is a race that can happen when multiple agents are trying - // to upload the same file. The other upload already succeeded so - // we don't need to upload it again. - u.retry.Add(fileID, struct{}{}) - return nil - } - if status.Code(err) == codes.InvalidArgument { - // This will never succeed, no need to retry. - u.retry.Add(fileID, struct{}{}) - return nil - } - return err - } - level.Debug(u.logger).Log("msg", "InitiateUpload", "res", fmt.Sprintf("%+v", initiateUploadResp)) - if initiateUploadResp.UploadInstructions == nil { - u.retry.Add(fileID, struct{}{}) - return nil - } + // Step 4: Stream chunks. + reader := bufio.NewReader(r) + buffer := make([]byte, ChunkSize) + var bytesSent uint64 - instructions := initiateUploadResp.UploadInstructions - var uploadedBytes uint64 - switch instructions.UploadStrategy { - case debuginfopb.UploadInstructions_UPLOAD_STRATEGY_SIGNED_URL: - if err := u.uploadViaSignedURL(ctx, instructions.SignedUrl, r, size); err != nil { - return err + for { + n, err := reader.Read(buffer) + if err == io.EOF { + break } - uploadedBytes = uint64(size) - case debuginfopb.UploadInstructions_UPLOAD_STRATEGY_GRPC: - var err error - uploadedBytes, err = NewGrpcUploadClient(client).Upload(ctx, instructions, r) if err != nil { - return err + return fmt.Errorf("read next chunk (%d bytes sent so far): %w", bytesSent, err) } - default: - // No clue what to do with this upload strategy. - level.Warn(u.logger).Log("msg", "unknown upload strategy", "strategy", instructions.UploadStrategy) - u.retry.Add(fileID, struct{}{}) - return nil - } - u.uploadRequestBytes.Add(float64(uploadedBytes)) + if err := stream.Send(&debuginfov1alpha1.UploadRequest{ + Data: &debuginfov1alpha1.UploadRequest_Chunk{ + Chunk: &debuginfov1alpha1.UploadChunk{ + Chunk: buffer[:n], + }, + }, + }); err != nil { + return fmt.Errorf("send chunk (%d bytes sent so far): %w", bytesSent, err) + } + bytesSent += uint64(n) + } - _, err = client.MarkUploadFinished(ctx, &debuginfopb.MarkUploadFinishedRequest{ - BuildId: buildID, - UploadId: initiateUploadResp.UploadInstructions.UploadId, - }) - if err != nil { - level.Debug(u.logger).Log("msg", "upload failed", "file_name", fileName, "build_id", buildID, "err", err) - return err + // Step 5: Close the send side to signal EOF. + if err := stream.CloseRequest(); err != nil { + return fmt.Errorf("close send: %w", err) } - level.Debug(u.logger).Log("msg", "upload succeeded", "file_name", fileName, "build_id", buildID, "bytes", uploadedBytes) + u.uploadRequestBytes.Add(float64(bytesSent)) + level.Debug(l).Log("msg", "upload succeeded", "bytes", bytesSent) u.retry.Add(fileID, struct{}{}) return nil } @@ -457,32 +417,3 @@ func readAtCloserSize(logger log.Logger, r process.ReadAtCloser) (int64, error) return stat.Size(), nil } - -// uploadViaSignedURL uploads the reader to the signed URL. -func (u *ParcaSymbolUploader) uploadViaSignedURL(ctx context.Context, url string, r io.Reader, size int64) error { - // Client is closing the reader if the reader is also closer. - // We need to wrap the reader to avoid this. - // We want to have total control over the reader. - r = bufio.NewReader(r) - req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, r) - if err != nil { - return fmt.Errorf("create request: %w", err) - } - - req.ContentLength = size - resp, err := u.httpClient.Do(req) - if err != nil { - return fmt.Errorf("do upload request: %w", err) - } - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - }() - - if resp.StatusCode/100 != 2 { - data, _ := io.ReadAll(resp.Body) - return fmt.Errorf("unexpected status code: %d, msg: %s", resp.StatusCode, string(data)) - } - - return nil -} diff --git a/internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader_test.go b/internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader_test.go new file mode 100644 index 00000000000..307343dcc23 --- /dev/null +++ b/internal/component/pyroscope/ebpf/reporter/parca/reporter/pyroscope_uploader_test.go @@ -0,0 +1,432 @@ +//go:build linux && (arm64 || amd64) + +package reporter + +import ( + "bytes" + "context" + "math/rand" + "net/http/httptest" + "os" + "sync" + "testing" + "time" + + "connectrpc.com/connect" + "github.com/go-kit/log" + debuginfov1alpha1 "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/process" + "go.uber.org/atomic" +) + +// mockDebuginfoHandler implements debuginfov1alpha1connect.DebuginfoServiceHandler. +type mockDebuginfoHandler struct { + uploadFunc func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error +} + +func (m *mockDebuginfoHandler) Upload(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + return m.uploadFunc(ctx, stream) +} + +// mockReadAtCloser wraps bytes.Reader to implement process.ReadAtCloser + Stater. +type mockReadAtCloser struct { + *bytes.Reader + size int64 +} + +func (m *mockReadAtCloser) Close() error { return nil } + +func (m *mockReadAtCloser) Stat() (os.FileInfo, error) { + return &mockFileInfo{size: m.size}, nil +} + +type mockFileInfo struct { + os.FileInfo + size int64 +} + +func (m *mockFileInfo) Size() int64 { return m.size } +func (m *mockFileInfo) IsDir() bool { return false } +func (m *mockFileInfo) Name() string { return "mock" } + +func newMockReadAtCloser(data []byte) func() (process.ReadAtCloser, error) { + return func() (process.ReadAtCloser, error) { + return &mockReadAtCloser{ + Reader: bytes.NewReader(data), + size: int64(len(data)), + }, nil + } +} + +func newTestUploader(t *testing.T) (*PyroscopeSymbolUploader, prometheus.Counter) { + t.Helper() + counter := prometheus.NewCounter(prometheus.CounterOpts{Name: "test_upload_bytes"}) + u, err := NewPyroscopeSymbolUploader( + log.NewNopLogger(), + 1024, // cacheSize + false, // stripTextSection + 64, // queueSize + 1, // workerNum + t.TempDir(), + counter, + ) + require.NoError(t, err) + return u, counter +} + +func startMockServer(t *testing.T, handler *mockDebuginfoHandler) debuginfov1alpha1connect.DebuginfoServiceClient { + t.Helper() + _, h := debuginfov1alpha1connect.NewDebuginfoServiceHandler(handler) + // Bidi streaming requires HTTP/2, so use TLS test server. + server := httptest.NewUnstartedServer(h) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return debuginfov1alpha1connect.NewDebuginfoServiceClient(server.Client(), server.URL) +} + +func counterValue(c prometheus.Counter) float64 { + m := &dto.Metric{} + _ = c.(prometheus.Metric).Write(m) + return m.GetCounter().GetValue() +} + +func TestMapShrink(t *testing.T) { + tr := newInProgressTracker(0.2) + r := rand.New(rand.NewSource(0)) + + items := make([]libpf.FileID, 100) + for i := 0; i < 100; i++ { + items[i] = libpf.NewFileID( + r.Uint64(), + r.Uint64(), + ) + + tr.GetOrAdd(items[i]) + } + + if tr.maxSizeSeen != 100 { + t.Errorf("expected 100, got %d", tr.maxSizeSeen) + } + + for i := 0; i < 10; i++ { + tr.Remove(items[i]) + } + + if tr.maxSizeSeen != 100 { + t.Errorf("expected 100, got %d", tr.maxSizeSeen) + } + + for i := 10; i < 20; i++ { + tr.Remove(items[i]) + } + + if tr.maxSizeSeen != 83 { + t.Errorf("expected 83, got %d", tr.maxSizeSeen) + } + + // adding up to 83 doesn't change anything + for i := 10; i < 13; i++ { + tr.GetOrAdd(items[i]) + } + + if tr.maxSizeSeen != 83 { + t.Errorf("expected 83, got %d", tr.maxSizeSeen) + } + + // adding 84th item should increases the max size + tr.GetOrAdd(items[13]) + + if tr.maxSizeSeen != 84 { + t.Errorf("expected 84, got %d", tr.maxSizeSeen) + } +} + +// receiveAllChunks drains chunk messages from a bidi stream, returning the concatenated data. +func receiveAllChunks(stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) ([]byte, int) { + var data []byte + chunks := 0 + for { + req, err := stream.Receive() + if err != nil { + break + } + if chunk := req.GetChunk(); chunk != nil { + chunks++ + data = append(data, chunk.GetChunk()...) + } + } + return data, chunks +} + +type uploadResult struct { + buildID string + fileName string + fileType debuginfov1alpha1.FileMetadata_Type + data []byte + chunks int +} + +func acceptUploadHandler(t *testing.T, resultCh chan<- uploadResult) *mockDebuginfoHandler { + t.Helper() + return &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + req, err := stream.Receive() + if err != nil { + return err + } + init := req.GetInit() + + if err := stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: true, + }, + }, + }); err != nil { + return err + } + + data, chunks := receiveAllChunks(stream) + resultCh <- uploadResult{ + buildID: init.GetFile().GetGnuBuildId(), + fileName: init.GetFile().GetName(), + fileType: init.GetFile().GetType(), + data: data, + chunks: chunks, + } + return nil + }, + } +} + +func TestAttemptUpload_Success(t *testing.T) { + fileData := []byte("hello debuginfo world") + resultCh := make(chan uploadResult, 1) + client := startMockServer(t, acceptUploadHandler(t, resultCh)) + u, counter := newTestUploader(t) + fileID := libpf.NewFileID(1, 2) + + err := u.attemptUpload(context.Background(), client, fileID, "test.so", "abc123", newMockReadAtCloser(fileData)) + require.NoError(t, err) + + select { + case res := <-resultCh: + require.Equal(t, "abc123", res.buildID) + require.Equal(t, "test.so", res.fileName) + require.Equal(t, fileData, res.data) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for server to receive upload") + } + require.Equal(t, float64(len(fileData)), counterValue(counter)) +} + +func TestAttemptUpload_ServerDeclinesUpload(t *testing.T) { + handler := &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + if _, err := stream.Receive(); err != nil { + return err + } + return stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: false, + Reason: "already exists", + }, + }, + }) + }, + } + + client := startMockServer(t, handler) + u, counter := newTestUploader(t) + fileID := libpf.NewFileID(3, 4) + + err := u.attemptUpload(context.Background(), client, fileID, "test.so", "def456", newMockReadAtCloser([]byte("data"))) + require.NoError(t, err) + require.Equal(t, float64(0), counterValue(counter)) + + // Verify the fileID was cached — a second Upload call should be skipped. + _, cached := u.retry.Get(fileID) + require.True(t, cached, "fileID should be in retry cache after declined upload") +} + +func TestAttemptUpload_UploadInProgress(t *testing.T) { + handler := &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + if _, err := stream.Receive(); err != nil { + return err + } + return stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: false, + Reason: ReasonUploadInProgress, + }, + }, + }) + }, + } + + client := startMockServer(t, handler) + u, _ := newTestUploader(t) + fileID := libpf.NewFileID(5, 6) + + err := u.attemptUpload(context.Background(), client, fileID, "test.so", "ghi789", newMockReadAtCloser([]byte("data"))) + require.NoError(t, err) + + // Should be cached with limited lifetime (not permanent). + _, cached := u.retry.Get(fileID) + require.True(t, cached, "fileID should be in retry cache for in-progress reason") +} + +func TestAttemptUpload_EmptyBuildID(t *testing.T) { + var receivedFile *debuginfov1alpha1.FileMetadata + + handler := &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + req, err := stream.Receive() + if err != nil { + return err + } + receivedFile = req.GetInit().GetFile() + return stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: false, + }, + }, + }) + }, + } + + client := startMockServer(t, handler) + u, _ := newTestUploader(t) + fileID := libpf.NewFileID(7, 8) + + // Pass empty buildID — GnuBuildId should be empty, OtelFileId should have the fileID. + err := u.attemptUpload(context.Background(), client, fileID, "test.so", "", newMockReadAtCloser([]byte("data"))) + require.NoError(t, err) + require.Equal(t, "", receivedFile.GetGnuBuildId()) + require.Equal(t, fileID.StringNoQuotes(), receivedFile.GetOtelFileId()) +} + +func TestAttemptUpload_LargeFile_MultipleChunks(t *testing.T) { + // Create data larger than ChunkSize (3MB) to force multiple chunks. + dataSize := ChunkSize*2 + 1024*512 // ~6.5MB → 3 chunks + fileData := make([]byte, dataSize) + for i := range fileData { + fileData[i] = byte(i % 256) + } + + resultCh := make(chan uploadResult, 1) + client := startMockServer(t, acceptUploadHandler(t, resultCh)) + u, counter := newTestUploader(t) + fileID := libpf.NewFileID(9, 10) + + err := u.attemptUpload(context.Background(), client, fileID, "big.so", "build1", newMockReadAtCloser(fileData)) + require.NoError(t, err) + + select { + case res := <-resultCh: + require.True(t, res.chunks >= 3, "expected at least 3 chunks, got %d", res.chunks) + require.Equal(t, fileData, res.data) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for server to receive upload") + } + require.Equal(t, float64(dataSize), counterValue(counter)) +} + +func TestUpload_Dedup(t *testing.T) { + uploadCount := atomic.NewInt32(0) + + handler := &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + uploadCount.Add(1) + if _, err := stream.Receive(); err != nil { + return err + } + if err := stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: true, + }, + }, + }); err != nil { + return err + } + // Drain chunks. + for { + _, err := stream.Receive() + if err != nil { + break + } + } + return nil + }, + } + + client := startMockServer(t, handler) + u, _ := newTestUploader(t) + fileID := libpf.NewFileID(11, 12) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = u.Run(ctx) + }() + + // Enqueue same fileID twice quickly. + u.Upload(ctx, client, fileID, "test.so", "build1", newMockReadAtCloser([]byte("data"))) + u.Upload(ctx, client, fileID, "test.so", "build1", newMockReadAtCloser([]byte("data"))) + + // Wait for worker to process. + time.Sleep(500 * time.Millisecond) + cancel() + wg.Wait() + + require.Equal(t, int32(1), uploadCount.Load(), "expected exactly 1 upload, second should be deduped") +} + +func TestUpload_WorkerProcessesQueue(t *testing.T) { + fileData := []byte("worker-test-data") + resultCh := make(chan uploadResult, 1) + client := startMockServer(t, acceptUploadHandler(t, resultCh)) + u, counter := newTestUploader(t) + fileID := libpf.NewFileID(13, 14) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = u.Run(ctx) + }() + + // Enqueue via Upload (goes through queue → worker → attemptUpload). + u.Upload(ctx, client, fileID, "worker.so", "build-worker", newMockReadAtCloser(fileData)) + + // Wait for the upload to complete on the server side. + select { + case res := <-resultCh: + require.Equal(t, fileData, res.data) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for upload") + } + + cancel() + wg.Wait() + + require.Equal(t, float64(len(fileData)), counterValue(counter)) +} diff --git a/internal/component/pyroscope/enrich/enrich.go b/internal/component/pyroscope/enrich/enrich.go index ecf8e920ad4..f516d32512e 100644 --- a/internal/component/pyroscope/enrich/enrich.go +++ b/internal/component/pyroscope/enrich/enrich.go @@ -5,12 +5,12 @@ import ( "context" "sync" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/pyroscope" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/prometheus/prometheus/model/labels" ) @@ -206,6 +206,6 @@ func (e *enrichAppendable) Upload(j debuginfo.UploadJob) { e.component.fanout.Upload(j) } -func (e *enrichAppendable) Client() debuginfogrpc.DebuginfoServiceClient { - return e.component.fanout.Client() +func (e *enrichAppendable) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + return e.component.fanout.DebugInfoClients() } diff --git a/internal/component/pyroscope/java/loop_test.go b/internal/component/pyroscope/java/loop_test.go index 1cb92f59a61..509f29ccbba 100644 --- a/internal/component/pyroscope/java/loop_test.go +++ b/internal/component/pyroscope/java/loop_test.go @@ -10,11 +10,11 @@ import ( "testing" "time" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/go-kit/log" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/pyroscope" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/mock" @@ -43,7 +43,7 @@ func (m *mockAppendable) Upload(j debuginfo.UploadJob) { } -func (m *mockAppendable) Client() debuginfogrpc.DebuginfoServiceClient { +func (m *mockAppendable) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { return nil } diff --git a/internal/component/pyroscope/receive_http/debuginfo.go b/internal/component/pyroscope/receive_http/debuginfo.go index 44b8dce517e..c0726df1408 100644 --- a/internal/component/pyroscope/receive_http/debuginfo.go +++ b/internal/component/pyroscope/receive_http/debuginfo.go @@ -2,98 +2,160 @@ package receive_http import ( "context" - "errors" + "fmt" "io" + "sync" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" - debuginfopb "buf.build/gen/go/parca-dev/parca/protocolbuffers/go/parca/debuginfo/v1alpha1" - "github.com/gorilla/mux" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "connectrpc.com/connect" + debuginfov1alpha1 "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" ) -var errNoDebugInfoClient = status.Error(codes.Unavailable, "no debug info client available") - -//nolint:unused -func (c *Component) mountDebugInfo(router *mux.Router) { - c.grpcServer = NewGrpcServer(c.server.Config()) - debuginfogrpc.RegisterDebuginfoServiceServer(c.grpcServer, c) - const ( - DebuginfoService_Upload_FullMethodName = "/parca.debuginfo.v1alpha1.DebuginfoService/Upload" - DebuginfoService_ShouldInitiateUpload_FullMethodName = "/parca.debuginfo.v1alpha1.DebuginfoService/ShouldInitiateUpload" - DebuginfoService_InitiateUpload_FullMethodName = "/parca.debuginfo.v1alpha1.DebuginfoService/InitiateUpload" - DebuginfoService_MarkUploadFinished_FullMethodName = "/parca.debuginfo.v1alpha1.DebuginfoService/MarkUploadFinished" - ) - router.PathPrefix(DebuginfoService_Upload_FullMethodName).Handler(c.grpcServer) - router.PathPrefix(DebuginfoService_ShouldInitiateUpload_FullMethodName).Handler(c.grpcServer) - router.PathPrefix(DebuginfoService_InitiateUpload_FullMethodName).Handler(c.grpcServer) - router.PathPrefix(DebuginfoService_MarkUploadFinished_FullMethodName).Handler(c.grpcServer) -} - -func (c *Component) getDebugInfoClient() debuginfogrpc.DebuginfoServiceClient { +func (c *Component) getDebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { c.mut.Lock() defer c.mut.Unlock() + var clients []debuginfov1alpha1connect.DebuginfoServiceClient for _, appendable := range c.appendables { - if client := appendable.Client(); client != nil { - return client - } + clients = append(clients, appendable.DebugInfoClients()...) } - return nil + return clients } -func (c *Component) Upload(stream grpc.ClientStreamingServer[debuginfopb.UploadRequest, debuginfopb.UploadResponse]) error { - client := c.getDebugInfoClient() - if client == nil { - return errNoDebugInfoClient +// Upload implements debuginfov1alpha1connect.DebuginfoServiceHandler. +// It fans out the upload to all downstream Connect clients. +func (c *Component) Upload(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + clients := c.getDebugInfoClients() + if len(clients) == 0 { + return connect.NewError(connect.CodeUnavailable, fmt.Errorf("no debug info clients available")) } - upstreamContext, upstreamCancel := context.WithCancel(stream.Context()) - defer upstreamCancel() - upstreamStream, err := client.Upload(upstreamContext) + // Step 1: Receive the init request from the caller. + initReq, err := stream.Receive() if err != nil { - return err + return fmt.Errorf("receive init request: %w", err) + } + if initReq.GetInit() == nil { + return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("expected init request")) } - for { - req, err := stream.Recv() - if errors.Is(err, io.EOF) { - resp, err := upstreamStream.CloseAndRecv() - if err != nil { - return err - } - return stream.SendAndClose(resp) + // Step 2: For each downstream client, open a bidi stream and send the init request. + type downstreamState struct { + stream *connect.BidiStreamForClient[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse] + should bool + reason string + } + + downstreams := make([]*downstreamState, 0, len(clients)) + anyShouldUpload := false + + for _, client := range clients { + ds := &downstreamState{} + ds.stream = client.Upload(ctx) + + // Send init request. + if err := ds.stream.Send(initReq); err != nil { + _ = ds.stream.CloseRequest() + _ = ds.stream.CloseResponse() + continue } + + // Receive init response. + resp, err := ds.stream.Receive() if err != nil { - return err + _ = ds.stream.CloseRequest() + _ = ds.stream.CloseResponse() + continue } - if err := upstreamStream.Send(req); err != nil { - return err + initResp := resp.GetInit() + if initResp == nil { + _ = ds.stream.CloseRequest() + _ = ds.stream.CloseResponse() + continue + } + + ds.should = initResp.ShouldInitiateUpload + ds.reason = initResp.Reason + + if ds.should { + anyShouldUpload = true + } else { + // Close streams for clients that don't need upload. + _ = ds.stream.CloseRequest() + _ = ds.stream.CloseResponse() + } + + downstreams = append(downstreams, ds) + } + + // Step 3: Send the merged response back to the caller. + mergedReason := "" + if !anyShouldUpload { + for _, ds := range downstreams { + if ds.reason != "" { + mergedReason = ds.reason + break + } } } -} -func (c *Component) ShouldInitiateUpload(ctx context.Context, request *debuginfopb.ShouldInitiateUploadRequest) (*debuginfopb.ShouldInitiateUploadResponse, error) { - client := c.getDebugInfoClient() - if client == nil { - return nil, errNoDebugInfoClient + if err := stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: anyShouldUpload, + Reason: mergedReason, + }, + }, + }); err != nil { + // Clean up all open streams. + for _, ds := range downstreams { + if ds.should { + _ = ds.stream.CloseRequest() + _ = ds.stream.CloseResponse() + } + } + return fmt.Errorf("send init response: %w", err) } - return client.ShouldInitiateUpload(ctx, request) -} -func (c *Component) InitiateUpload(ctx context.Context, request *debuginfopb.InitiateUploadRequest) (*debuginfopb.InitiateUploadResponse, error) { - client := c.getDebugInfoClient() - if client == nil { - return nil, errNoDebugInfoClient + if !anyShouldUpload { + return nil } - return client.InitiateUpload(ctx, request) -} -func (c *Component) MarkUploadFinished(ctx context.Context, request *debuginfopb.MarkUploadFinishedRequest) (*debuginfopb.MarkUploadFinishedResponse, error) { - client := c.getDebugInfoClient() - if client == nil { - return nil, errNoDebugInfoClient + // Step 4: Stream chunks from caller to all approved downstream clients. + activeStreams := make([]*downstreamState, 0) + for _, ds := range downstreams { + if ds.should { + activeStreams = append(activeStreams, ds) + } } - return client.MarkUploadFinished(ctx, request) + + for { + req, err := stream.Receive() + if err == io.EOF || err != nil { + break + } + + chunk := req.GetChunk() + if chunk == nil { + continue + } + + // Fan-out chunk to all active downstream clients concurrently. + var wg sync.WaitGroup + for _, ds := range activeStreams { + wg.Add(1) + go func(ds *downstreamState) { + defer wg.Done() + _ = ds.stream.Send(req) + }(ds) + } + wg.Wait() + } + + // Step 5: Close the send side of all active downstream streams to signal EOF. + for _, ds := range activeStreams { + _ = ds.stream.CloseRequest() + } + + return nil } diff --git a/internal/component/pyroscope/receive_http/grpc.go b/internal/component/pyroscope/receive_http/grpc.go deleted file mode 100644 index 589d00d47e3..00000000000 --- a/internal/component/pyroscope/receive_http/grpc.go +++ /dev/null @@ -1,35 +0,0 @@ -package receive_http - -import ( - "github.com/grafana/dskit/server" - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" -) - -func NewGrpcServer(cfg server.Config) *grpc.Server { - grpcKeepAliveOptions := keepalive.ServerParameters{ - MaxConnectionIdle: cfg.GRPCServerMaxConnectionIdle, - MaxConnectionAge: cfg.GRPCServerMaxConnectionAge, - MaxConnectionAgeGrace: cfg.GRPCServerMaxConnectionAgeGrace, - Time: cfg.GRPCServerTime, - Timeout: cfg.GRPCServerTimeout, - } - - grpcKeepAliveEnforcementPolicy := keepalive.EnforcementPolicy{ - MinTime: cfg.GRPCServerMinTimeBetweenPings, - PermitWithoutStream: cfg.GRPCServerPingWithoutStreamAllowed, - } - - grpcOptions := []grpc.ServerOption{ - grpc.KeepaliveParams(grpcKeepAliveOptions), - grpc.KeepaliveEnforcementPolicy(grpcKeepAliveEnforcementPolicy), - grpc.MaxRecvMsgSize(cfg.GRPCServerMaxRecvMsgSize), - grpc.MaxSendMsgSize(cfg.GRPCServerMaxSendMsgSize), - grpc.MaxConcurrentStreams(uint32(cfg.GRPCServerMaxConcurrentStreams)), - grpc.NumStreamWorkers(uint32(cfg.GRPCServerNumWorkers)), - } - - grpcOptions = append(grpcOptions, cfg.GRPCOptions...) - - return grpc.NewServer(grpcOptions...) -} diff --git a/internal/component/pyroscope/receive_http/receive_http.go b/internal/component/pyroscope/receive_http/receive_http.go index 237283de489..5c57f5e9331 100644 --- a/internal/component/pyroscope/receive_http/receive_http.go +++ b/internal/component/pyroscope/receive_http/receive_http.go @@ -21,6 +21,7 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/util" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" "github.com/grafana/pyroscope/api/gen/proto/go/push/v1/pushv1connect" typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" @@ -28,7 +29,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/model/labels" "go.opentelemetry.io/otel/trace" - "google.golang.org/grpc" ) func init() { @@ -61,7 +61,6 @@ type Component struct { serverConfig *fnet.HTTPConfig uncheckedCollector *util.UncheckedCollector appendables []pyroscope.Appendable - grpcServer *grpc.Server mut sync.Mutex logger log.Logger tracer trace.Tracer @@ -106,7 +105,7 @@ func (c *Component) Update(args component.Arguments) error { func (c *Component) update(args component.Arguments) (bool, error) { shutdown := false newArgs := args.(Arguments) - // required for debug info upload over grpc over http2 over http server port + // required for debug info upload over connect over http2 over http server port if newArgs.Server.HTTP.HTTP2 == nil { newArgs.Server.HTTP.HTTP2 = &fnet.HTTP2Config{} } @@ -147,6 +146,9 @@ func (c *Component) update(args component.Arguments) (bool, error) { // mount connect go pushv1 pathPush, handlePush := pushv1connect.NewPusherServiceHandler(c) router.PathPrefix(pathPush).Handler(handlePush).Methods(http.MethodPost) + + // mount connect debuginfo upload handler + debuginfov1alpha1connect.RegisterDebuginfoServiceHandler(router, c) }) } @@ -304,10 +306,6 @@ func (c *Component) handleIngest(w http.ResponseWriter, r *http.Request) { } func (c *Component) shutdownServer() { - if c.grpcServer != nil { - c.grpcServer.GracefulStop() - c.grpcServer = nil - } if c.server != nil { c.server.StopAndShutdown() c.server = nil diff --git a/internal/component/pyroscope/receive_http/receive_http_test.go b/internal/component/pyroscope/receive_http/receive_http_test.go index c1e614ac219..913de499b91 100644 --- a/internal/component/pyroscope/receive_http/receive_http_test.go +++ b/internal/component/pyroscope/receive_http/receive_http_test.go @@ -7,13 +7,15 @@ import ( "errors" "fmt" "net/http" + "net/http/httptest" "net/url" "sync" "testing" "time" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "connectrpc.com/connect" + debuginfov1alpha1 "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "go.opentelemetry.io/otel/trace/noop" "github.com/phayes/freeport" @@ -547,7 +549,7 @@ func (a *testAppender) AppendIngest(_ context.Context, profile *pyroscope.Incomi return a.appendErr } -func (a *testAppender) Client() debuginfogrpc.DebuginfoServiceClient { +func (a *testAppender) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { return nil } @@ -690,3 +692,307 @@ func TestAPIToAlloySamples(t *testing.T) { }) } } + +// --- debuginfo proxy tests --- + +// mockDebuginfoHandler implements debuginfov1alpha1connect.DebuginfoServiceHandler for testing. +type mockDebuginfoHandler struct { + uploadFunc func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error +} + +func (m *mockDebuginfoHandler) Upload(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + return m.uploadFunc(ctx, stream) +} + +// downstreamResult captures what a mock downstream server received. +type downstreamResult struct { + initFile *debuginfov1alpha1.FileMetadata + data []byte + chunks int +} + +// startMockDownstream creates a TLS httptest server (HTTP/2) running a mock debuginfo handler. +// shouldUpload controls whether the server accepts uploads. +// resultCh receives the captured result after the handler finishes. +func startMockDownstream(t *testing.T, shouldUpload bool, resultCh chan<- downstreamResult) debuginfov1alpha1connect.DebuginfoServiceClient { + t.Helper() + handler := &mockDebuginfoHandler{ + uploadFunc: func(ctx context.Context, stream *connect.BidiStream[debuginfov1alpha1.UploadRequest, debuginfov1alpha1.UploadResponse]) error { + req, err := stream.Receive() + if err != nil { + return err + } + + if err := stream.Send(&debuginfov1alpha1.UploadResponse{ + Data: &debuginfov1alpha1.UploadResponse_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadResponse{ + ShouldInitiateUpload: shouldUpload, + }, + }, + }); err != nil { + return err + } + + res := downstreamResult{ + initFile: req.GetInit().GetFile(), + } + + if shouldUpload { + for { + chunkReq, err := stream.Receive() + if err != nil { + break + } + if chunk := chunkReq.GetChunk(); chunk != nil { + res.chunks++ + res.data = append(res.data, chunk.GetChunk()...) + } + } + } + + resultCh <- res + return nil + }, + } + + _, h := debuginfov1alpha1connect.NewDebuginfoServiceHandler(handler) + server := httptest.NewUnstartedServer(h) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return debuginfov1alpha1connect.NewDebuginfoServiceClient(server.Client(), server.URL) +} + +// debuginfoAppendable is a test pyroscope.Appendable that returns Connect debuginfo clients. +type debuginfoAppendable struct { + clients []debuginfov1alpha1connect.DebuginfoServiceClient +} + +func (d *debuginfoAppendable) Appender() pyroscope.Appender { return d } +func (d *debuginfoAppendable) Append(_ context.Context, _ labels.Labels, _ []*pyroscope.RawSample) error { + return nil +} +func (d *debuginfoAppendable) AppendIngest(_ context.Context, _ *pyroscope.IncomingProfile) error { + return nil +} +func (d *debuginfoAppendable) Upload(_ debuginfo.UploadJob) {} +func (d *debuginfoAppendable) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + return d.clients +} + +// startProxyServer creates a Component with the given appendables and serves its +// debuginfo Upload handler via an HTTP/2 TLS test server. Returns a Connect client. +func startProxyServer(t *testing.T, appendables []pyroscope.Appendable) debuginfov1alpha1connect.DebuginfoServiceClient { + t.Helper() + comp := &Component{ + appendables: appendables, + } + _, h := debuginfov1alpha1connect.NewDebuginfoServiceHandler(comp) + server := httptest.NewUnstartedServer(h) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return debuginfov1alpha1connect.NewDebuginfoServiceClient(server.Client(), server.URL) +} + +// sendUploadViaProxy opens a bidi stream to the proxy client, sends an init request, reads the +// init response, and if accepted, streams fileData as chunks. +func sendUploadViaProxy(t *testing.T, proxyClient debuginfov1alpha1connect.DebuginfoServiceClient, fileData []byte) (bool, error) { + t.Helper() + stream := proxyClient.Upload(context.Background()) + + // Send init. + err := stream.Send(&debuginfov1alpha1.UploadRequest{ + Data: &debuginfov1alpha1.UploadRequest_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadRequest{ + File: &debuginfov1alpha1.FileMetadata{ + GnuBuildId: "test-build-id", + Name: "test.so", + Type: debuginfov1alpha1.FileMetadata_TYPE_EXECUTABLE_FULL, + }, + }, + }, + }) + if err != nil { + return false, fmt.Errorf("send init: %w", err) + } + + // Receive init response. + resp, err := stream.Receive() + if err != nil { + return false, fmt.Errorf("receive init response: %w", err) + } + + initResp := resp.GetInit() + if initResp == nil { + return false, fmt.Errorf("expected init response") + } + + if !initResp.ShouldInitiateUpload { + _ = stream.CloseRequest() + return false, nil + } + + // Stream chunks. + chunkSize := 1024 + for offset := 0; offset < len(fileData); offset += chunkSize { + end := offset + chunkSize + if end > len(fileData) { + end = len(fileData) + } + if err := stream.Send(&debuginfov1alpha1.UploadRequest{ + Data: &debuginfov1alpha1.UploadRequest_Chunk{ + Chunk: &debuginfov1alpha1.UploadChunk{ + Chunk: fileData[offset:end], + }, + }, + }); err != nil { + return true, fmt.Errorf("send chunk: %w", err) + } + } + + _ = stream.CloseRequest() + return true, nil +} + +func TestDebugInfoProxy_SingleEndpoint_AcceptsUpload(t *testing.T) { + resultCh := make(chan downstreamResult, 1) + dsClient := startMockDownstream(t, true, resultCh) + + appendable := &debuginfoAppendable{clients: []debuginfov1alpha1connect.DebuginfoServiceClient{dsClient}} + proxyClient := startProxyServer(t, []pyroscope.Appendable{appendable}) + + fileData := []byte("hello proxy debuginfo upload test data") + accepted, err := sendUploadViaProxy(t, proxyClient, fileData) + require.NoError(t, err) + require.True(t, accepted) + + select { + case res := <-resultCh: + require.Equal(t, "test-build-id", res.initFile.GetGnuBuildId()) + require.Equal(t, "test.so", res.initFile.GetName()) + require.Equal(t, fileData, res.data) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for downstream to receive upload") + } +} + +func TestDebugInfoProxy_MultipleEndpoints_AllAccept(t *testing.T) { + const numEndpoints = 3 + resultChs := make([]chan downstreamResult, numEndpoints) + clients := make([]debuginfov1alpha1connect.DebuginfoServiceClient, numEndpoints) + + for i := 0; i < numEndpoints; i++ { + resultChs[i] = make(chan downstreamResult, 1) + clients[i] = startMockDownstream(t, true, resultChs[i]) + } + + appendable := &debuginfoAppendable{clients: clients} + proxyClient := startProxyServer(t, []pyroscope.Appendable{appendable}) + + fileData := []byte("multi-endpoint-test-data-for-all-accepting-servers") + accepted, err := sendUploadViaProxy(t, proxyClient, fileData) + require.NoError(t, err) + require.True(t, accepted) + + for i := 0; i < numEndpoints; i++ { + select { + case res := <-resultChs[i]: + require.Equal(t, "test-build-id", res.initFile.GetGnuBuildId(), "endpoint %d", i) + require.Equal(t, fileData, res.data, "endpoint %d data mismatch", i) + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for downstream %d", i) + } + } +} + +func TestDebugInfoProxy_MultipleEndpoints_AllDecline(t *testing.T) { + const numEndpoints = 3 + resultChs := make([]chan downstreamResult, numEndpoints) + clients := make([]debuginfov1alpha1connect.DebuginfoServiceClient, numEndpoints) + + for i := 0; i < numEndpoints; i++ { + resultChs[i] = make(chan downstreamResult, 1) + clients[i] = startMockDownstream(t, false, resultChs[i]) + } + + appendable := &debuginfoAppendable{clients: clients} + proxyClient := startProxyServer(t, []pyroscope.Appendable{appendable}) + + fileData := []byte("should-not-be-sent") + accepted, err := sendUploadViaProxy(t, proxyClient, fileData) + require.NoError(t, err) + require.False(t, accepted, "proxy should decline when all endpoints decline") + + // Verify all downstreams received the init but no chunks. + for i := 0; i < numEndpoints; i++ { + select { + case res := <-resultChs[i]: + require.Equal(t, "test-build-id", res.initFile.GetGnuBuildId(), "endpoint %d", i) + require.Nil(t, res.data, "endpoint %d should not have received chunks", i) + require.Equal(t, 0, res.chunks, "endpoint %d should not have received chunks", i) + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for downstream %d", i) + } + } +} + +func TestDebugInfoProxy_MultipleEndpoints_SomeAccept(t *testing.T) { + // 3 endpoints: [0]=decline, [1]=accept, [2]=accept + accepts := []bool{false, true, true} + resultChs := make([]chan downstreamResult, len(accepts)) + clients := make([]debuginfov1alpha1connect.DebuginfoServiceClient, len(accepts)) + + for i, shouldAccept := range accepts { + resultChs[i] = make(chan downstreamResult, 1) + clients[i] = startMockDownstream(t, shouldAccept, resultChs[i]) + } + + appendable := &debuginfoAppendable{clients: clients} + proxyClient := startProxyServer(t, []pyroscope.Appendable{appendable}) + + fileData := []byte("partial-accept-test-data") + accepted, err := sendUploadViaProxy(t, proxyClient, fileData) + require.NoError(t, err) + require.True(t, accepted, "proxy should accept when at least one endpoint accepts") + + for i, shouldAccept := range accepts { + select { + case res := <-resultChs[i]: + require.Equal(t, "test-build-id", res.initFile.GetGnuBuildId(), "endpoint %d", i) + if shouldAccept { + require.Equal(t, fileData, res.data, "accepting endpoint %d data mismatch", i) + } else { + require.Nil(t, res.data, "declining endpoint %d should not receive chunks", i) + } + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for downstream %d", i) + } + } +} + +func TestDebugInfoProxy_NoEndpoints(t *testing.T) { + // No downstream clients at all. + appendable := &debuginfoAppendable{clients: nil} + proxyClient := startProxyServer(t, []pyroscope.Appendable{appendable}) + stream := proxyClient.Upload(context.Background()) + + err := stream.Send(&debuginfov1alpha1.UploadRequest{ + Data: &debuginfov1alpha1.UploadRequest_Init{ + Init: &debuginfov1alpha1.ShouldInitiateUploadRequest{ + File: &debuginfov1alpha1.FileMetadata{ + GnuBuildId: "test", + Name: "test.so", + }, + }, + }, + }) + if err != nil { + // Error on send is acceptable — server may reject immediately. + return + } + + _, err = stream.Receive() + require.Error(t, err) + require.Equal(t, connect.CodeUnavailable, connect.CodeOf(err)) +} diff --git a/internal/component/pyroscope/relabel/relabel.go b/internal/component/pyroscope/relabel/relabel.go index 994c2b955c7..9f62160a1b0 100644 --- a/internal/component/pyroscope/relabel/relabel.go +++ b/internal/component/pyroscope/relabel/relabel.go @@ -17,8 +17,8 @@ import ( "reflect" "sync" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "go.uber.org/atomic" "github.com/grafana/alloy/internal/component" @@ -309,6 +309,6 @@ func (c *Component) Upload(j debuginfo.UploadJob) { c.fanout.Upload(j) } -func (c *Component) Client() debuginfogrpc.DebuginfoServiceClient { - return c.fanout.Client() +func (c *Component) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + return c.fanout.DebugInfoClients() } diff --git a/internal/component/pyroscope/relabel/relabel_test.go b/internal/component/pyroscope/relabel/relabel_test.go index 09aeeed5264..635ed3561f8 100644 --- a/internal/component/pyroscope/relabel/relabel_test.go +++ b/internal/component/pyroscope/relabel/relabel_test.go @@ -7,12 +7,12 @@ import ( "sync" "testing" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component" alloy_relabel "github.com/grafana/alloy/internal/component/common/relabel" "github.com/grafana/alloy/internal/component/pyroscope" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" "github.com/grafana/alloy/internal/util" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/grafana/pyroscope/api/model/labelset" "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" @@ -463,7 +463,7 @@ func (t *TestAppender) Upload(j debuginfo.UploadJob) { } -func (t *TestAppender) Client() debuginfogrpc.DebuginfoServiceClient { +func (t *TestAppender) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { return nil } diff --git a/internal/component/pyroscope/write/debuginfo/common.go b/internal/component/pyroscope/write/debuginfo/common.go index 74b358eeee1..85291fabe09 100644 --- a/internal/component/pyroscope/write/debuginfo/common.go +++ b/internal/component/pyroscope/write/debuginfo/common.go @@ -4,21 +4,19 @@ import ( "context" "sync" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/go-kit/log" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "github.com/prometheus/client_golang/prometheus" - "google.golang.org/grpc" ) type Appender interface { // Upload dispatches the job recursively to each of the nested children, down to each write component, - //down to Client and therefore parca's uploader + // down to Client and therefore the uploader. Upload(j UploadJob) - // Client returns the direct grpc client of the first nested child (down to write component) - // this is a best-effort support for the proxy case - we do not support fan-out for proxy only one-to-one - // forwarding to the first write endpoint. - Client() debuginfogrpc.DebuginfoServiceClient + // DebugInfoClients returns ALL Connect debuginfo clients from all nested children. + // This is used by the receive_http proxy to fan-out uploads to all downstream endpoints. + DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient } type Arguments struct { @@ -30,52 +28,41 @@ type Arguments struct { WorkerNum int `alloy:"worker_num,attr,optional"` } -func NewClient(logger log.Logger, newClient func() (*grpc.ClientConn, error), +func NewClient(logger log.Logger, connectClient debuginfov1alpha1connect.DebuginfoServiceClient, metric prometheus.Counter, dataPath string) *Client { return &Client{ - newClient: newClient, - metric: metric, - dataPath: dataPath, - - logger: logger, - uploaderChan: make(chan *uploader, 1), + connectClient: connectClient, + metric: metric, + dataPath: dataPath, + logger: logger, + uploaderChan: make(chan *uploader, 1), } } -// Client is per write-endpoint debug info upload client +// Client is per write-endpoint debug info upload client. // This structure serves two purposes: -// - return the grpc client to the receive_http component +// - return the connect client to the receive_http component for proxying // - perform the debug info upload from the current host by the ebpf profiler request type Client struct { - logger log.Logger - newClient func() (*grpc.ClientConn, error) - clientOnce sync.Once - cc *grpc.ClientConn - client debuginfogrpc.DebuginfoServiceClient - uploaderOnce sync.Once - uploader *uploader - uploaderChan chan *uploader - metric prometheus.Counter - dataPath string + logger log.Logger + connectClient debuginfov1alpha1connect.DebuginfoServiceClient + uploaderOnce sync.Once + uploader *uploader + uploaderChan chan *uploader + metric prometheus.Counter + dataPath string } -func (c *Client) Client() debuginfogrpc.DebuginfoServiceClient { - c.clientOnce.Do(func() { - var err error - c.cc, err = c.newClient() - if err != nil { - _ = level.Error(c.logger).Log("msg", "error initializing debuginfo client", "err", err) - return - } - c.client = debuginfogrpc.NewDebuginfoServiceClient(c.cc) - }) - return c.client +func (c *Client) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + if c.connectClient != nil { + return []debuginfov1alpha1connect.DebuginfoServiceClient{c.connectClient} + } + return nil } func (c *Client) Upload(j UploadJob) { - cc := c.Client() - if cc == nil { + if c.connectClient == nil { return } c.uploaderOnce.Do(func() { @@ -92,15 +79,10 @@ func (c *Client) Upload(j UploadJob) { return } - c.uploader.upload(cc, j) + c.uploader.upload(c.connectClient, j) } func (c *Client) Run(ctx context.Context) error { - defer func() { - if c.cc != nil { - _ = c.cc.Close() - } - }() select { case <-ctx.Done(): return ctx.Err() diff --git a/internal/component/pyroscope/write/debuginfo/stub.go b/internal/component/pyroscope/write/debuginfo/stub.go index 7711cf1c3e4..9a793c750eb 100644 --- a/internal/component/pyroscope/write/debuginfo/stub.go +++ b/internal/component/pyroscope/write/debuginfo/stub.go @@ -5,7 +5,7 @@ package debuginfo import ( "context" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" ) type UploadJob struct { @@ -19,7 +19,7 @@ func (c *Client) newUploader(j UploadJob) (*uploader, error) { type uploader struct { } -func (u uploader) upload(c debuginfogrpc.DebuginfoServiceClient, j UploadJob) { +func (u uploader) upload(c debuginfov1alpha1connect.DebuginfoServiceClient, j UploadJob) { // no-op } diff --git a/internal/component/pyroscope/write/debuginfo/parca.go b/internal/component/pyroscope/write/debuginfo/upload.go similarity index 72% rename from internal/component/pyroscope/write/debuginfo/parca.go rename to internal/component/pyroscope/write/debuginfo/upload.go index 521e4902a70..9dc9f44550c 100644 --- a/internal/component/pyroscope/write/debuginfo/parca.go +++ b/internal/component/pyroscope/write/debuginfo/upload.go @@ -5,8 +5,8 @@ package debuginfo import ( "context" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" "github.com/grafana/alloy/internal/component/pyroscope/ebpf/reporter/parca/reporter" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/process" ) @@ -14,15 +14,15 @@ import ( type UploadJob struct { FrameMappingFileData libpf.FrameMappingFileData Open func() (process.ReadAtCloser, error) - // InitArguments is the structure used to create a new parca uploader - // it is passed as the job field to have the configuration in the ebpf component instead of write component, + // InitArguments is the structure used to create a new uploader. + // It is passed as the job field to have the configuration in the ebpf component instead of write component, // to not confuse users. InitArguments Arguments } func (c *Client) newUploader(j UploadJob) (*uploader, error) { args := j.InitArguments - u, err := reporter.NewParcaSymbolUploader( + u, err := reporter.NewPyroscopeSymbolUploader( c.logger, args.CacheSize, args.StripTextSection, @@ -38,10 +38,10 @@ func (c *Client) newUploader(j UploadJob) (*uploader, error) { } type uploader struct { - u *reporter.ParcaSymbolUploader + u *reporter.PyroscopeSymbolUploader } -func (u *uploader) upload(c debuginfogrpc.DebuginfoServiceClient, j UploadJob) { +func (u *uploader) upload(c debuginfov1alpha1connect.DebuginfoServiceClient, j UploadJob) { u.u.Upload(context.Background(), c, j.FrameMappingFileData.FileID, diff --git a/internal/component/pyroscope/write/debuginfo_client.go b/internal/component/pyroscope/write/debuginfo_client.go deleted file mode 100644 index ece2c4ab5a5..00000000000 --- a/internal/component/pyroscope/write/debuginfo_client.go +++ /dev/null @@ -1,103 +0,0 @@ -package write - -import ( - "context" - "crypto/tls" - "encoding/base64" - "fmt" - "net/url" - "os" - "strings" - - commonconfig "github.com/prometheus/common/config" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" -) - -func newDebugInfoGRPCClient(u *url.URL, e *EndpointOptions) (*grpc.ClientConn, error) { - var creds credentials.TransportCredentials - var auth *basicAuthCredential - switch u.Scheme { - case "http": - creds = insecure.NewCredentials() - case "https": - if promTLSConfig := e.HTTPClientConfig.TLSConfig.Convert(); promTLSConfig != nil { - tlsConf, err := commonconfig.NewTLSConfig(promTLSConfig) - if err != nil { - return nil, err - } - creds = credentials.NewTLS(tlsConf) - } else { - creds = credentials.NewTLS(&tls.Config{}) - } - var err error - if auth, err = newGrpcBasicAuthCredentials(e); err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("unsupported scheme: %s", u.Scheme) - } - - opts := []grpc.DialOption{ - grpc.WithTransportCredentials(creds), - } - if auth != nil { - opts = append(opts, grpc.WithPerRPCCredentials(auth)) - } - target := u.Host - if u.Port() == "" { - defaultPort := "80" - if u.Scheme == "https" { - defaultPort = "443" - } - target = fmt.Sprintf("%s:%s", u.Hostname(), defaultPort) - } - cc, err := grpc.NewClient(target, opts...) - if err != nil { - return nil, err - } - - return cc, nil -} - -func newGrpcBasicAuthCredentials(e *EndpointOptions) (*basicAuthCredential, error) { - auth := e.HTTPClientConfig.BasicAuth - if auth == nil || auth.Username == "" { - return nil, nil - } - if auth.Password != "" { - return &basicAuthCredential{ - username: auth.Username, - password: string(auth.Password), - }, nil - } - if auth.PasswordFile != "" { - passwordBytes, err := os.ReadFile(auth.PasswordFile) - if err != nil { - return nil, err - } - return &basicAuthCredential{ - username: auth.Username, - password: strings.TrimSpace(string(passwordBytes)), - }, nil - } - return nil, nil -} - -type basicAuthCredential struct { - username string - password string -} - -func (b *basicAuthCredential) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - auth := b.username + ":" + b.password - encodedAuth := base64.StdEncoding.EncodeToString([]byte(auth)) - return map[string]string{ - "authorization": "Basic " + encodedAuth, - }, nil -} - -func (b *basicAuthCredential) RequireTransportSecurity() bool { - return true -} diff --git a/internal/component/pyroscope/write/write.go b/internal/component/pyroscope/write/write.go index 98354af3e0f..4b2c36186d1 100644 --- a/internal/component/pyroscope/write/write.go +++ b/internal/component/pyroscope/write/write.go @@ -3,9 +3,11 @@ package write import ( "bytes" "context" + "crypto/tls" "errors" "fmt" "io" + "net" "net/http" "net/url" "path" @@ -14,7 +16,8 @@ import ( "sync" "time" - debuginfogrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc" + "golang.org/x/net/http2" + "connectrpc.com/connect" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -23,6 +26,7 @@ import ( "github.com/grafana/alloy/internal/component/pyroscope/util" "github.com/grafana/alloy/internal/component/pyroscope/write/debuginfo" "github.com/grafana/dskit/backoff" + "github.com/grafana/pyroscope/api/gen/proto/go/debuginfo/v1alpha1/debuginfov1alpha1connect" pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" "github.com/grafana/pyroscope/api/gen/proto/go/push/v1/pushv1connect" typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" @@ -35,7 +39,6 @@ import ( "go.opentelemetry.io/contrib/propagators/jaeger" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" - "google.golang.org/grpc" ) var ( @@ -226,14 +229,12 @@ type fanOutClient struct { uploaderWg sync.WaitGroup } -func (f *fanOutClient) Client() debuginfogrpc.DebuginfoServiceClient { +func (f *fanOutClient) DebugInfoClients() []debuginfov1alpha1connect.DebuginfoServiceClient { + var clients []debuginfov1alpha1connect.DebuginfoServiceClient for _, client := range f.debugInfos { - cl := client.Client() - if cl != nil { - return cl - } + clients = append(clients, client.DebugInfoClients()...) } - return nil + return clients } func (f *fanOutClient) Upload(j debuginfo.UploadJob) { @@ -265,10 +266,6 @@ func newFanOut(logger log.Logger, tracer trace.Tracer, config Arguments, metrics ingestClients := make(map[*EndpointOptions]*http.Client) for i, endpoint := range config.Endpoints { - u, err := url.Parse(endpoint.URL) - if err != nil { - return nil, err - } if endpoint.Headers == nil { endpoint.Headers = map[string]string{} } @@ -286,9 +283,12 @@ func newFanOut(logger log.Logger, tracer trace.Tracer, config Arguments, metrics ingestClients[endpoint] = httpClient endpointDataPath := filepath.Join(dataPath, fmt.Sprintf("endpoint-%d", i)) - debugInfo := debuginfo.NewClient(logger, func() (*grpc.ClientConn, error) { - return newDebugInfoGRPCClient(u, endpoint) - }, metrics.debugInfoUploadBytes, endpointDataPath) + // Bidi streaming (debuginfo upload) requires HTTP/2. For HTTPS this + // works via ALPN; for plain HTTP we need an h2c-capable transport + // because receive_http and pyroscope both serve h2c. + debuginfoHTTPClient := newH2CClient(endpoint.URL, httpClient) + connectClient := debuginfov1alpha1connect.NewDebuginfoServiceClient(debuginfoHTTPClient, endpoint.URL) + debugInfo := debuginfo.NewClient(logger, connectClient, metrics.debugInfoUploadBytes, endpointDataPath) debugInfos = append(debugInfos, debugInfo) } @@ -755,6 +755,34 @@ func validateLabels(lbls labels.Labels) error { return err } +// newH2CClient returns an HTTP client for Connect bidi streaming. +// For HTTPS endpoints, the base client is returned as-is (HTTP/2 via ALPN). +// For plain HTTP endpoints, returns a client with h2c transport because bidi +// streaming requires HTTP/2 and both receive_http and pyroscope serve h2c. +// +// Note: the Go standard library and prometheus/common/config do not support +// h2c natively (same pattern used in internal/service/cluster/cluster.go). +// The h2c transport reuses the base client's settings where possible. +func newH2CClient(endpointURL string, base *http.Client) *http.Client { + u, err := url.Parse(endpointURL) + if err != nil || u.Scheme == "https" { + return base + } + return &http.Client{ + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { + // Plain TCP dial for h2c (same approach as cluster.go). + var d net.Dialer + return d.DialContext(ctx, network, addr) + }, + }, + Timeout: base.Timeout, + CheckRedirect: base.CheckRedirect, + Jar: base.Jar, + } +} + func configureTracing(config Arguments, httpClient *http.Client) { if config.Tracing.JaegerPropagator || config.Tracing.TraceContextPropagator { var propagators []propagation.TextMapPropagator