Skip to content

feat(auth): add OpenTelemetry gRPC and HTTP wrappers for T4 tracing #14133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
quartzmo merged 19 commits into from
Mar 12, 2026
Merged

feat(auth): add OpenTelemetry gRPC and HTTP wrappers for T4 tracing #14133

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
9186633
feat(auth): add OpenTelemetry gRPC and HTTP wrappers for T4 tracing
quartzmo Feb 4, 2026
b501608
fixes for Gemini comments
quartzmo Mar 9, 2026
f4fd2ab
fix for Gemini comment in httptransport
quartzmo Mar 9, 2026
90baac0
update gcp.resource.name to gcp.resource.destination.id
quartzmo Mar 9, 2026
372467b
update grpc.status to rpc.response.status_code
quartzmo Mar 9, 2026
3dc0cb8
update rpc.system to rpc.system.name
quartzmo Mar 9, 2026
1e86183
update gcp.grpc.resend_count to http.request.resend_count
quartzmo Mar 9, 2026
8e646be
populate static known attributes from generator values including url.…
quartzmo Mar 9, 2026
2276c04
add internal function doc
quartzmo Mar 9, 2026
dabf100
update gax-go/v2 dep to 2.18.0
quartzmo Mar 10, 2026
448149c
add knownKeys test and docs
quartzmo Mar 10, 2026
29ae1a4
set rpc.response.status_code OK when HandleRPC succeeds
quartzmo Mar 10, 2026
1556dc4
improve error handling in HTTP RoundTrip
quartzmo Mar 10, 2026
40572b0
goimports -w .
quartzmo Mar 10, 2026
baee448
Merge branch 'main' into auth-grpc-tracing-t4
quartzmo Mar 11, 2026
dec02c8
delegate staticAttrs to NewTransport
quartzmo Mar 11, 2026
70cbded
improve error handling in grpc HandleRPC
quartzmo Mar 11, 2026
9f6ee63
fix usage of status.FromError
quartzmo Mar 12, 2026
c92ecff
check local ctx.Err() first for grpc error.type
quartzmo Mar 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions auth/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/google/go-cmp v0.7.0
github.com/google/s2a-go v0.1.9
github.com/googleapis/enterprise-certificate-proxy v0.3.14
github.com/googleapis/gax-go/v2 v2.17.0
github.com/googleapis/gax-go/v2 v2.18.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0
go.opentelemetry.io/otel v1.40.0
Expand All @@ -28,9 +28,10 @@ require (
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
)
18 changes: 12 additions & 6 deletions auth/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
github.com/googleapis/gax-go/v2 v2.18.0 h1:jxP5Uuo3bxm3M6gGtV94P4lliVetoCB4Wk2x8QA86LI=
github.com/googleapis/gax-go/v2 v2.18.0/go.mod h1:uSzZN4a356eRG985CzJ3WfbFSpqkLTjsnhWGJR6EwrE=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down Expand Up @@ -58,8 +58,8 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
Expand All @@ -70,8 +70,14 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d h1:vsOm753cOAMkt76efriTCDKjpCbK18XGHMJHo0JUKhc=
google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:0oz9d7g9QLSdv9/lgbIjowW1JoxMbxmBVNe8i6tORJI=
google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d h1:EocjzKLywydp5uZ5tJ79iP6Q0UjDnyiHkGRWxuPBP8s=
google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:48U2I+QQUYhsFrg2SY6r+nJzeOtjey7j//WBESw+qyQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU=
google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
Expand Down
97 changes: 95 additions & 2 deletions auth/grpctransport/grpctransport.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,26 @@ import (
"log/slog"
"net/http"
"os"
"strconv"
"strings"

"cloud.google.com/go/auth"
"cloud.google.com/go/auth/credentials"
"cloud.google.com/go/auth/internal"
"cloud.google.com/go/auth/internal/transport"
"cloud.google.com/go/auth/internal/transport/headers"
"github.com/googleapis/gax-go/v2"
"github.com/googleapis/gax-go/v2/callctx"
"github.com/googleapis/gax-go/v2/internallog"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
grpccreds "google.golang.org/grpc/credentials"
grpcinsecure "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/stats"
"google.golang.org/grpc/status"
)

const (
Expand Down Expand Up @@ -198,7 +207,7 @@ type InternalOptions struct {
// service.
DefaultScopes []string
// SkipValidation bypasses validation on Options. It should only be used
// internally for clients that needs more control over their transport.
// internally for clients that need more control over their transport.
SkipValidation bool
// TelemetryAttributes specifies a map of telemetry attributes to be added
// to all OpenTelemetry signals, such as tracing and metrics, for purposes
Expand Down Expand Up @@ -430,5 +439,89 @@ func addOpenTelemetryStatsHandler(dialOpts []grpc.DialOption, opts *Options) []g
if opts.DisableTelemetry {
return dialOpts
}
return append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
if !gax.IsFeatureEnabled("TRACING") {
return append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler()))
}
var staticAttrs []attribute.KeyValue
if opts.InternalOptions != nil {
staticAttrs = transport.StaticTelemetryAttributes(opts.InternalOptions.TelemetryAttributes)
}
otelOpts := []otelgrpc.Option{
otelgrpc.WithSpanAttributes(staticAttrs...),
}
return append(dialOpts, grpc.WithStatsHandler(&otelHandler{
Handler: otelgrpc.NewClientHandler(otelOpts...),
}))
}

// otelHandler is a wrapper around the OpenTelemetry gRPC client handler that
// adds custom Google Cloud-specific attributes to spans and metrics.
type otelHandler struct {
stats.Handler
}

// TagRPC intercepts the RPC start to extract dynamic attributes like resource
// name and retry count from the outgoing context metadata and attach them to
// the current span.
func (h *otelHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
ctx = h.Handler.TagRPC(ctx, info)
span := trace.SpanFromContext(ctx)
if !span.IsRecording() {
return ctx
}
var attrs []attribute.KeyValue
if resName, ok := callctx.TelemetryFromContext(ctx, "resource_name"); ok {
attrs = append(attrs, attribute.String("gcp.resource.destination.id", resName))
}
if resendCountStr, ok := callctx.TelemetryFromContext(ctx, "resend_count"); ok {
if count, err := strconv.Atoi(resendCountStr); err == nil {
attrs = append(attrs, attribute.Int("gcp.grpc.resend_count", count))
}
}
Comment thread
quartzmo marked this conversation as resolved.
if len(attrs) > 0 {
span.SetAttributes(attrs...)
}
return ctx
}

// HandleRPC intercepts the RPC completion to capture and format error-related
// attributes ensuring they conform to Google Cloud observability standards.
func (h *otelHandler) HandleRPC(ctx context.Context, s stats.RPCStats) {
end, ok := s.(*stats.End)
if !ok {
h.Handler.HandleRPC(ctx, s)
return
}
span := trace.SpanFromContext(ctx)
if !span.IsRecording() {
h.Handler.HandleRPC(ctx, s)
return
}

var attrs []attribute.KeyValue
if end.Error != nil {
// Extract status code and message
st, _ := status.FromError(end.Error)
var errorType string
switch st.Code() {
case codes.DeadlineExceeded:
errorType = "CLIENT_TIMEOUT"
case codes.Canceled:
errorType = "CLIENT_CANCELLED"
Comment thread
quartzmo marked this conversation as resolved.
default:
errorType = strings.ToUpper(st.Code().String())
}
attrs = []attribute.KeyValue{
attribute.String("error.type", errorType),
attribute.String("status.message", st.Message()),
attribute.String("rpc.response.status_code", strings.ToUpper(st.Code().String())),
attribute.String("exception.type", fmt.Sprintf("%T", end.Error)),
}
} else {
attrs = []attribute.KeyValue{
attribute.String("rpc.response.status_code", "OK"),
}
}
span.SetAttributes(attrs...)
h.Handler.HandleRPC(ctx, s)
}
Loading
Loading