From d2ad136d4774f0bef7fa710ddc774063ada2aa68 Mon Sep 17 00:00:00 2001 From: Luc Talatinian <102624213+lucix-aws@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:58:52 -0400 Subject: [PATCH] add tracing and metrics support to generated clients (#538) --- .../76820480d9aa4b7389f4b8ca928a1522.json | 8 + .../8a8ce2aaa0b84978a2b874ca2f90d021.json | 8 + .../93daa8ff4ccc4cb8a8a02aef737bb20c.json | 8 + .../amazon/smithy/go/codegen/GoWriter.java | 8 +- .../smithy/go/codegen/ServiceGenerator.java | 89 +++++++-- .../smithy/go/codegen/SmithyGoDependency.java | 2 + .../auth/GetIdentityMiddlewareGenerator.java | 26 ++- .../ResolveAuthSchemeMiddlewareGenerator.java | 10 +- .../auth/SignRequestMiddlewareGenerator.java | 32 ++- .../EndpointMiddlewareGenerator.java | 20 +- .../HttpBindingProtocolGenerator.java | 19 ++ .../integration/HttpRpcProtocolGenerator.java | 19 ++ .../integration/ObservabilityOptions.java | 84 ++++++++ .../integration/OperationMetricsStruct.java | 187 +++++++++++++++++ .../go/codegen/integration/TracingSpans.java | 166 ++++++++++++++++ .../middleware/BuildStepMiddleware.java | 44 ++++ .../middleware/DeserializeStepMiddleware.java | 44 ++++ .../middleware/FinalizeStepMiddleware.java | 44 ++++ .../middleware/InitializeStepMiddleware.java | 44 ++++ .../middleware/OperationMiddleware.java | 91 +++++++++ .../middleware/SerializeStepMiddleware.java | 44 ++++ ...mithy.go.codegen.integration.GoIntegration | 3 + metrics/metrics.go | 136 +++++++++++++ metrics/nop.go | 67 +++++++ metrics/smithyotelmetrics/async.go | 62 ++++++ metrics/smithyotelmetrics/attribute.go | 67 +++++++ metrics/smithyotelmetrics/doc.go | 40 ++++ metrics/smithyotelmetrics/float64.go | 43 ++++ metrics/smithyotelmetrics/go.mod | 11 + metrics/smithyotelmetrics/go.sum | 20 ++ metrics/smithyotelmetrics/int64.go | 41 ++++ metrics/smithyotelmetrics/metrics.go | 188 ++++++++++++++++++ metrics/smithyotelmetrics/option.go | 23 +++ middleware/context.go | 41 ++++ properties.go | 19 +- tracing/context.go | 96 +++++++++ tracing/context_test.go | 64 ++++++ tracing/nop.go | 32 +++ tracing/smithyoteltracing/adapt.go | 140 +++++++++++++ tracing/smithyoteltracing/adapt_test.go | 52 +++++ tracing/smithyoteltracing/attribute.go | 67 +++++++ tracing/smithyoteltracing/attribute_test.go | 46 +++++ tracing/smithyoteltracing/doc.go | 69 +++++++ tracing/smithyoteltracing/go.mod | 11 + tracing/smithyoteltracing/go.sum | 14 ++ tracing/tracing.go | 95 +++++++++ transport/http/client.go | 18 ++ 47 files changed, 2417 insertions(+), 45 deletions(-) create mode 100644 .changelog/76820480d9aa4b7389f4b8ca928a1522.json create mode 100644 .changelog/8a8ce2aaa0b84978a2b874ca2f90d021.json create mode 100644 .changelog/93daa8ff4ccc4cb8a8a02aef737bb20c.json create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/ObservabilityOptions.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/OperationMetricsStruct.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/TracingSpans.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/BuildStepMiddleware.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/DeserializeStepMiddleware.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/FinalizeStepMiddleware.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/InitializeStepMiddleware.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/OperationMiddleware.java create mode 100644 codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/SerializeStepMiddleware.java create mode 100644 metrics/metrics.go create mode 100644 metrics/nop.go create mode 100644 metrics/smithyotelmetrics/async.go create mode 100644 metrics/smithyotelmetrics/attribute.go create mode 100644 metrics/smithyotelmetrics/doc.go create mode 100644 metrics/smithyotelmetrics/float64.go create mode 100644 metrics/smithyotelmetrics/go.mod create mode 100644 metrics/smithyotelmetrics/go.sum create mode 100644 metrics/smithyotelmetrics/int64.go create mode 100644 metrics/smithyotelmetrics/metrics.go create mode 100644 metrics/smithyotelmetrics/option.go create mode 100644 middleware/context.go create mode 100644 tracing/context.go create mode 100644 tracing/context_test.go create mode 100644 tracing/nop.go create mode 100644 tracing/smithyoteltracing/adapt.go create mode 100644 tracing/smithyoteltracing/adapt_test.go create mode 100644 tracing/smithyoteltracing/attribute.go create mode 100644 tracing/smithyoteltracing/attribute_test.go create mode 100644 tracing/smithyoteltracing/doc.go create mode 100644 tracing/smithyoteltracing/go.mod create mode 100644 tracing/smithyoteltracing/go.sum create mode 100644 tracing/tracing.go diff --git a/.changelog/76820480d9aa4b7389f4b8ca928a1522.json b/.changelog/76820480d9aa4b7389f4b8ca928a1522.json new file mode 100644 index 000000000..8dffbfff2 --- /dev/null +++ b/.changelog/76820480d9aa4b7389f4b8ca928a1522.json @@ -0,0 +1,8 @@ +{ + "id": "76820480-d9aa-4b73-89f4-b8ca928a1522", + "type": "feature", + "description": "Add tracing and metrics APIs, and builtin instrumentation for both, in generated clients.", + "modules": [ + "." + ] +} \ No newline at end of file diff --git a/.changelog/8a8ce2aaa0b84978a2b874ca2f90d021.json b/.changelog/8a8ce2aaa0b84978a2b874ca2f90d021.json new file mode 100644 index 000000000..d0c0f117b --- /dev/null +++ b/.changelog/8a8ce2aaa0b84978a2b874ca2f90d021.json @@ -0,0 +1,8 @@ +{ + "id": "8a8ce2aa-a0b8-4978-a2b8-74ca2f90d021", + "type": "release", + "description": "Initial release of `smithyotelmetrics` module, which is used to adapt an OpenTelemetry SDK meter provider to be used with Smithy clients.", + "modules": [ + "metrics/smithyotelmetrics" + ] +} diff --git a/.changelog/93daa8ff4ccc4cb8a8a02aef737bb20c.json b/.changelog/93daa8ff4ccc4cb8a8a02aef737bb20c.json new file mode 100644 index 000000000..34fb0a72c --- /dev/null +++ b/.changelog/93daa8ff4ccc4cb8a8a02aef737bb20c.json @@ -0,0 +1,8 @@ +{ + "id": "93daa8ff-4ccc-4cb8-a8a0-2aef737bb20c", + "type": "release", + "description": "Initial release of `smithyoteltracing` module, which is used to adapt an OpenTelemetry SDK tracer provider to be used with Smithy clients.", + "modules": [ + "tracing/smithyoteltracing" + ] +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java index ec1c41f49..028d81131 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java @@ -59,7 +59,7 @@ public final class GoWriter extends SymbolWriter { private static final Logger LOGGER = Logger.getLogger(GoWriter.class.getName()); private static final int DEFAULT_DOC_WRAP_LENGTH = 80; - private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]+)(:\\w)?"); + private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]\\.+)(:\\w)?"); private final String fullPackageName; private final boolean innerWriter; private final List buildTags = new ArrayList<>(); @@ -97,6 +97,12 @@ private void init() { putFormatter('W', new GoWritableInjector()); putFormatter('D', new GoDependencyFormatter()); + putContext("fmt.Sprintf", SmithyGoDependency.FMT.func("Sprintf")); + putContext("fmt.Errorf", SmithyGoDependency.FMT.func("Errorf")); + putContext("errors.As", SmithyGoDependency.ERRORS.func("As")); + putContext("context.Context", SmithyGoDependency.CONTEXT.func("Context")); + putContext("time.Now", SmithyGoDependency.TIME.func("Now")); + if (!innerWriter) { packageDocs = new GoWriter(this.fullPackageName, true); } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java index 3bacb2f03..38d084f3e 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java @@ -37,6 +37,7 @@ import software.amazon.smithy.go.codegen.integration.ClientMemberResolver; import software.amazon.smithy.go.codegen.integration.ConfigFieldResolver; import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.OperationMetricsStruct; import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.knowledge.ServiceIndex; @@ -94,6 +95,7 @@ public void run() { private GoWriter.Writable generate() { return GoWriter.ChainWritable.of( generateMetadata(), + generateObservabilityComponents(), generateClient(), generateNew(), generateGetOptions(), @@ -103,6 +105,22 @@ private GoWriter.Writable generate() { ).compose(); } + private GoWriter.Writable generateObservabilityComponents() { + return goTemplate(""" + $operationMetrics:W + + func operationTracer(p $tracerProvider:T) $tracer:T { + return p.Tracer($scope:S) + } + """, + Map.of( + "tracerProvider", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("TracerProvider"), + "tracer", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("Tracer"), + "scope", settings.getModuleName(), + "operationMetrics", new OperationMetricsStruct(settings.getModuleName()) + )); + } + private GoWriter.Writable generateMetadata() { var serviceId = settings.getService().toString(); for (var integration : integrations) { @@ -324,8 +342,16 @@ func resolveAuthSchemes(options *Options) { @SuppressWarnings("checkstyle:LineLength") private GoWriter.Writable generateInvokeOperation() { return goTemplate(""" - func (c *Client) invokeOperation(ctx $context:T, opID string, params interface{}, optFns []func(*Options), stackFns ...func($stack:P, Options) error) (result interface{}, metadata $metadata:T, err error) { - ctx = $clearStackValues:T(ctx) + $middleware:D $tracing:D + func (c *Client) invokeOperation( + ctx context.Context, opID string, params interface{}, optFns []func(*Options), stackFns ...func(*middleware.Stack, Options) error, + ) ( + result interface{}, metadata middleware.Metadata, err error, + ) { + ctx = middleware.ClearStackValues(ctx) + ctx = middleware.WithServiceID(ctx, ServiceID) + ctx = middleware.WithOperationName(ctx, opID) + $newStack:W options := c.options.Copy() $resolvers:W @@ -348,25 +374,61 @@ private GoWriter.Writable generateInvokeOperation() { } } - $newStackHandler:W - result, metadata, err = handler.Handle(ctx, params) + ctx, err = withOperationMetrics(ctx, options.MeterProvider) if err != nil { + return nil, metadata, err + } + + tracer := operationTracer(options.TracerProvider) + spanName := fmt.Sprintf("%s.%s", ServiceID, opID) + + ctx = tracing.WithOperationTracer(ctx, tracer) + + ctx, span := tracer.StartSpan(ctx, spanName, func (o *tracing.SpanOptions) { + o.Kind = tracing.SpanKindClient + o.Properties.Set("rpc.system", "aws-api") + o.Properties.Set("rpc.method", opID) + o.Properties.Set("rpc.service", ServiceID) + }) + endTimer := startMetricTimer(ctx, "client.call.duration") + defer endTimer() + defer span.End() + + handler := $newClientHandler:T(options.HTTPClient) + decorated := middleware.DecorateHandler(handler, stack) + result, metadata, err = decorated.Handle(ctx, params) + if err != nil { + span.SetProperty("exception.type", fmt.Sprintf("%T", err)) + span.SetProperty("exception.message", err.Error()) + + var aerr smithy.APIError + if $errors.As:T(err, &aerr) { + span.SetProperty("api.error_code", aerr.ErrorCode()) + span.SetProperty("api.error_message", aerr.ErrorMessage()) + span.SetProperty("api.error_fault", aerr.ErrorFault().String()) + } + err = &$operationError:T{ ServiceID: ServiceID, OperationName: opID, Err: err, } } + + span.SetProperty("error", err != nil) + if err == nil { + span.SetStatus(tracing.SpanStatusOK) + } else { + span.SetStatus(tracing.SpanStatusError) + } + return result, metadata, err } """, MapUtils.of( - "context", GoStdlibTypes.Context.Context, - "stack", SmithyGoTypes.Middleware.Stack, - "metadata", SmithyGoTypes.Middleware.Metadata, - "clearStackValues", SmithyGoTypes.Middleware.ClearStackValues, + "middleware", SmithyGoDependency.SMITHY_MIDDLEWARE, + "tracing", SmithyGoDependency.SMITHY_TRACING, "newStack", generateNewStack(), - "newStackHandler", generateNewStackHandler(), "operationError", SmithyGoTypes.Smithy.OperationError, "resolvers", GoWriter.ChainWritable.of( getConfigResolvers( @@ -379,7 +441,8 @@ private GoWriter.Writable generateInvokeOperation() { ConfigFieldResolver.Location.OPERATION, ConfigFieldResolver.Target.FINALIZATION ).map(this::generateConfigFieldResolver).toList() - ).compose() + ).compose(), + "newClientHandler", SmithyGoDependency.SMITHY_HTTP_TRANSPORT.func("NewClientHandler") )); } @@ -389,12 +452,6 @@ private GoWriter.Writable generateNewStack() { SmithyGoTypes.Middleware.NewStack, SmithyGoTypes.Transport.Http.NewStackRequest); } - private GoWriter.Writable generateNewStackHandler() { - ensureSupportedProtocol(); - return goTemplate("handler := $T($T(options.HTTPClient), stack)", - SmithyGoTypes.Middleware.DecorateHandler, SmithyGoTypes.Transport.Http.NewClientHandler); - } - private void ensureSupportedProtocol() { if (!applicationProtocol.isHttpProtocol()) { throw new UnsupportedOperationException( diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java index 9867e614e..8f873575f 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java @@ -76,6 +76,8 @@ public final class SmithyGoDependency { public static final GoDependency SMITHY_AUTH_BEARER = smithy("auth/bearer"); public static final GoDependency SMITHY_ENDPOINTS = smithy("endpoints", "smithyendpoints"); public static final GoDependency SMITHY_ENDPOINT_RULESFN = smithy("endpoints/private/rulesfn"); + public static final GoDependency SMITHY_TRACING = smithy("tracing"); + public static final GoDependency SMITHY_METRICS = smithy("metrics"); public static final GoDependency GO_JMESPATH = goJmespath(null); public static final GoDependency MATH = stdlib("math"); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java index e9604e432..8dca07542 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java @@ -21,6 +21,7 @@ import software.amazon.smithy.go.codegen.GoStdlibTypes; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.MiddlewareIdentifier; +import software.amazon.smithy.go.codegen.SmithyGoDependency; import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.integration.ProtocolGenerator; import software.amazon.smithy.utils.MapUtils; @@ -64,26 +65,39 @@ private GoWriter.Writable generateFields() { private GoWriter.Writable generateBody() { return goTemplate(""" - rscheme := getResolvedAuthScheme(ctx) + innerCtx, span := $startSpan:T(ctx, "GetIdentity") + defer span.End() + + rscheme := getResolvedAuthScheme(innerCtx) if rscheme == nil { - return out, metadata, $errorf:T("no resolved auth scheme") + return out, metadata, $fmt.Errorf:T("no resolved auth scheme") } resolver := rscheme.Scheme.IdentityResolver(m.options) if resolver == nil { - return out, metadata, $errorf:T("no identity resolver") + return out, metadata, $fmt.Errorf:T("no identity resolver") } - identity, err := resolver.GetIdentity(ctx, rscheme.IdentityProperties) + identity, err := timeOperationMetric(ctx, "client.call.resolve_identity_duration", + func() ($identity:T, error) { + return resolver.GetIdentity(innerCtx, rscheme.IdentityProperties) + }, + func (o $recordMetricOptions:P) { + o.Properties.Set("auth.scheme_id", rscheme.Scheme.SchemeID()) + }) if err != nil { - return out, metadata, $errorf:T("get identity: %w", err) + return out, metadata, $fmt.Errorf:T("get identity: %w", err) } ctx = setIdentity(ctx, identity) + + span.End() return next.HandleFinalize(ctx, in) """, MapUtils.of( - "errorf", GoStdlibTypes.Fmt.Errorf + "startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan"), + "identity", SmithyGoDependency.SMITHY_AUTH.interfaceSymbol("Identity"), + "recordMetricOptions", SmithyGoDependency.SMITHY_METRICS.struct("RecordMetricOptions") )); } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java index e0d6b80d2..588605f7f 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java @@ -22,6 +22,7 @@ import software.amazon.smithy.go.codegen.GoStdlibTypes; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.MiddlewareIdentifier; +import software.amazon.smithy.go.codegen.SmithyGoDependency; import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.integration.ProtocolGenerator; import software.amazon.smithy.utils.MapUtils; @@ -66,6 +67,9 @@ private GoWriter.Writable generateFields() { private GoWriter.Writable generateBody() { return goTemplate(""" + _, span := $3T(ctx, "ResolveAuthScheme") + defer span.End() + params := $1L(ctx, m.operation, getOperationInput(ctx), m.options) options, err := m.options.AuthSchemeResolver.ResolveAuthSchemes(ctx, params) if err != nil { @@ -78,10 +82,14 @@ private GoWriter.Writable generateBody() { } ctx = setResolvedAuthScheme(ctx, scheme) + + span.SetProperty("auth.scheme_id", scheme.Scheme.SchemeID()) + span.End() return next.HandleFinalize(ctx, in) """, AuthParametersResolverGenerator.FUNC_NAME, - GoStdlibTypes.Fmt.Errorf + GoStdlibTypes.Fmt.Errorf, + SmithyGoDependency.SMITHY_TRACING.func("StartSpan") ); } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java index ab5484560..9378ccffa 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java @@ -16,12 +16,12 @@ package software.amazon.smithy.go.codegen.auth; import static software.amazon.smithy.go.codegen.GoStackStepMiddlewareGenerator.createFinalizeStepMiddleware; -import static software.amazon.smithy.go.codegen.GoWriter.emptyGoTemplate; import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; import software.amazon.smithy.go.codegen.GoStdlibTypes; import software.amazon.smithy.go.codegen.GoWriter; import software.amazon.smithy.go.codegen.MiddlewareIdentifier; +import software.amazon.smithy.go.codegen.SmithyGoDependency; import software.amazon.smithy.go.codegen.SmithyGoTypes; import software.amazon.smithy.go.codegen.endpoints.EndpointMiddlewareGenerator; import software.amazon.smithy.go.codegen.integration.ProtocolGenerator; @@ -39,7 +39,7 @@ public SignRequestMiddlewareGenerator(ProtocolGenerator.GenerationContext contex public static GoWriter.Writable generateAddToProtocolFinalizers() { return goTemplate(""" - if err := stack.Finalize.Insert(&$L{}, $S, $T); err != nil { + if err := stack.Finalize.Insert(&$L{options: options}, $S, $T); err != nil { return $T("add $L: %w", err) } """, @@ -56,41 +56,53 @@ public GoWriter.Writable generate() { } private GoWriter.Writable generateFields() { - return emptyGoTemplate(); + return goTemplate(""" + options Options + """); } private GoWriter.Writable generateBody() { return goTemplate(""" + _, span := $startSpan:T(ctx, "SignRequest") + defer span.End() + req, ok := in.Request.($request:P) if !ok { - return out, metadata, $errorf:T("unexpected transport type %T", in.Request) + return out, metadata, $fmt.Errorf:T("unexpected transport type %T", in.Request) } rscheme := getResolvedAuthScheme(ctx) if rscheme == nil { - return out, metadata, $errorf:T("no resolved auth scheme") + return out, metadata, $fmt.Errorf:T("no resolved auth scheme") } identity := getIdentity(ctx) if identity == nil { - return out, metadata, $errorf:T("no identity") + return out, metadata, $fmt.Errorf:T("no identity") } signer := rscheme.Scheme.Signer() if signer == nil { - return out, metadata, $errorf:T("no signer") + return out, metadata, $fmt.Errorf:T("no signer") } - if err := signer.SignRequest(ctx, req, identity, rscheme.SignerProperties); err != nil { - return out, metadata, $errorf:T("sign request: %w", err) + _, err = timeOperationMetric(ctx, "client.call.signing_duration", func() (any, error) { + return nil, signer.SignRequest(ctx, req, identity, rscheme.SignerProperties) + }, func(o $recordMetricOptions:P) { + o.Properties.Set("auth.scheme_id", rscheme.Scheme.SchemeID()) + }) + if err != nil { + return out, metadata, $fmt.Errorf:T("sign request: %w", err) } + span.End() return next.HandleFinalize(ctx, in) """, MapUtils.of( // FUTURE(#458) protocol generator should specify the transport type "request", SmithyGoTypes.Transport.Http.Request, - "errorf", GoStdlibTypes.Fmt.Errorf + "startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan"), + "recordMetricOptions", SmithyGoDependency.SMITHY_METRICS.struct("RecordMetricOptions") )); } } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/endpoints/EndpointMiddlewareGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/endpoints/EndpointMiddlewareGenerator.java index 63adbac05..36bce2bd3 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/endpoints/EndpointMiddlewareGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/endpoints/EndpointMiddlewareGenerator.java @@ -17,6 +17,7 @@ import static software.amazon.smithy.go.codegen.GoStackStepMiddlewareGenerator.createFinalizeStepMiddleware; import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_TRACING; import software.amazon.smithy.go.codegen.GoStdlibTypes; import software.amazon.smithy.go.codegen.GoWriter; @@ -72,6 +73,9 @@ private GoWriter.Writable generateBody() { } return goTemplate(""" + _, span := $startSpan:T(ctx, "ResolveEndpoint") + defer span.End() + $pre:W $assertRequest:W @@ -84,9 +88,11 @@ private GoWriter.Writable generateBody() { $post:W + span.End() return next.HandleFinalize(ctx, in) """, MapUtils.of( + "startSpan", SMITHY_TRACING.func("StartSpan"), "pre", generatePreResolutionHooks(), "assertRequest", generateAssertRequest(), "assertResolver", generateAssertResolver(), @@ -127,23 +133,27 @@ private GoWriter.Writable generateAssertResolver() { private GoWriter.Writable generateResolveEndpoint() { return goTemplate(""" params := bindEndpointParams(ctx, getOperationInput(ctx), m.options) - endpt, err := m.options.EndpointResolverV2.ResolveEndpoint(ctx, *params) + endpt, err := timeOperationMetric(ctx, "client.call.resolve_endpoint_duration", + func() (smithyendpoints.Endpoint, error) { + return m.options.EndpointResolverV2.ResolveEndpoint(ctx, *params) + }) if err != nil { - return out, metadata, $1T("failed to resolve service endpoint, %w", err) + return out, metadata, $fmt.Errorf:T("failed to resolve service endpoint, %w", err) } + span.SetProperty("client.call.resolved_endpoint", endpt.URI.String()) + if endpt.URI.RawPath == "" && req.URL.RawPath != "" { endpt.URI.RawPath = endpt.URI.Path } req.URL.Scheme = endpt.URI.Scheme req.URL.Host = endpt.URI.Host - req.URL.Path = $2T(endpt.URI.Path, req.URL.Path) - req.URL.RawPath = $2T(endpt.URI.RawPath, req.URL.RawPath) + req.URL.Path = $1T(endpt.URI.Path, req.URL.Path) + req.URL.RawPath = $1T(endpt.URI.RawPath, req.URL.RawPath) for k := range endpt.Headers { req.Header.Set(k, endpt.Headers.Get(k)) } """, - GoStdlibTypes.Fmt.Errorf, SmithyGoTypes.Transport.Http.JoinPath); } diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpBindingProtocolGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpBindingProtocolGenerator.java index 82920b143..3049a0b2d 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpBindingProtocolGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpBindingProtocolGenerator.java @@ -15,6 +15,8 @@ package software.amazon.smithy.go.codegen.integration; +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_TRACING; import static software.amazon.smithy.go.codegen.integration.ProtocolUtils.requiresDocumentSerdeFunction; import java.util.Collection; @@ -250,6 +252,13 @@ private void generateOperationSerializerMiddleware(GenerationContext context, Op writer.addUseImports(SmithyGoDependency.SMITHY); writer.addUseImports(SmithyGoDependency.SMITHY_HTTP_BINDING); + writer.write(goTemplate(""" + _, span := $T(ctx, "OperationSerializer") + endTimer := startMetricTimer(ctx, "client.call.serialization_duration") + defer endTimer() + defer span.End() + """, SMITHY_TRACING.func("StartSpan"))); + // cast input request to smithy transport type, check for failures writer.write("request, ok := in.Request.($P)", requestType); writer.openBlock("if !ok {", "}", () -> { @@ -351,6 +360,8 @@ private void generateOperationSerializerMiddleware(GenerationContext context, Op writer.write("in.Request = request"); writer.write(""); + writer.write("endTimer()"); + writer.write("span.End()"); writer.write("return next.$L(ctx, in)", generator.getHandleMethodName()); }); } @@ -385,6 +396,13 @@ private void generateOperationDeserializerMiddleware(GenerationContext context, writer.write("if err != nil { return out, metadata, err }"); writer.write(""); + writer.write(goTemplate(""" + _, span := $T(ctx, "OperationDeserializer") + endTimer := startMetricTimer(ctx, "client.call.deserialization_duration") + defer endTimer() + defer span.End() + """, SMITHY_TRACING.func("StartSpan"))); + writer.write("response, ok := out.RawResponse.($P)", responseType); writer.openBlock("if !ok {", "}", () -> { writer.addUseImports(SmithyGoDependency.SMITHY); @@ -451,6 +469,7 @@ private void generateOperationDeserializerMiddleware(GenerationContext context, } writer.write(""); + writer.write("span.End()"); writer.write("return out, metadata, err"); }); goWriter.write(""); diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpRpcProtocolGenerator.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpRpcProtocolGenerator.java index f47492405..b8278cfbd 100644 --- a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpRpcProtocolGenerator.java +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpRpcProtocolGenerator.java @@ -15,6 +15,9 @@ package software.amazon.smithy.go.codegen.integration; +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_TRACING; + import java.util.Optional; import java.util.Set; import java.util.TreeSet; @@ -144,6 +147,13 @@ private void generateOperationSerializer(GenerationContext context, OperationSha writer.addUseImports(SmithyGoDependency.FMT); writer.addUseImports(SmithyGoDependency.SMITHY_HTTP_BINDING); + writer.write(goTemplate(""" + _, span := $T(ctx, "OperationSerializer") + endTimer := startMetricTimer(ctx, "client.call.serialization_duration") + defer endTimer() + defer span.End() + """, SMITHY_TRACING.func("StartSpan"))); + // TODO: refactor the http binding encoder to be split up into its component parts // This would allow most of this shared code to be split off into its own function // to reduce duplication, and potentially allowing it to be a static function. @@ -212,6 +222,8 @@ private void generateOperationSerializer(GenerationContext context, OperationSha writer.write("in.Request = request"); writer.write(""); + writer.write("endTimer()"); + writer.write("span.End()"); writer.write("return next.$L(ctx, in)", generator.getHandleMethodName()); }); @@ -330,6 +342,13 @@ private void generateOperationDeserializer(GenerationContext context, OperationS writer.write("if err != nil { return out, metadata, err }"); writer.write(""); + writer.write(goTemplate(""" + _, span := $T(ctx, "OperationDeserializer") + endTimer := startMetricTimer(ctx, "client.call.deserialization_duration") + defer endTimer() + defer span.End() + """, SMITHY_TRACING.func("StartSpan"))); + writer.write("response, ok := out.RawResponse.($P)", responseType); writer.openBlock("if !ok {", "}", () -> { writer.write(String.format("return out, metadata, &smithy.DeserializationError{Err: %s}", diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/ObservabilityOptions.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/ObservabilityOptions.java new file mode 100644 index 000000000..99166c62e --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/ObservabilityOptions.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.integration; + +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol; + +import java.util.List; +import software.amazon.smithy.go.codegen.GoCodegenContext; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Adds observability providers to client options. + */ +public class ObservabilityOptions implements GoIntegration { + private static final ConfigField TRACER_PROVIDER = ConfigField.builder() + .name("TracerProvider") + .type(SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("TracerProvider")) + .documentation("The client tracer provider.") + .build(); + + private static final ConfigField METER_PROVIDER = ConfigField.builder() + .name("MeterProvider") + .type(SmithyGoDependency.SMITHY_METRICS.interfaceSymbol("MeterProvider")) + .documentation("The client meter provider.") + .build(); + + private static final ConfigFieldResolver RESOLVE_TRACER_PROVIDER = ConfigFieldResolver.builder() + .resolver(buildPackageSymbol("resolveTracerProvider")) + .location(ConfigFieldResolver.Location.CLIENT) + .target(ConfigFieldResolver.Target.INITIALIZATION) + .build(); + + private static final ConfigFieldResolver RESOLVE_METER_PROVIDER = ConfigFieldResolver.builder() + .resolver(buildPackageSymbol("resolveMeterProvider")) + .location(ConfigFieldResolver.Location.CLIENT) + .target(ConfigFieldResolver.Target.INITIALIZATION) + .build(); + + @Override + public List getClientPlugins() { + return List.of( + RuntimeClientPlugin.builder() + .addConfigField(TRACER_PROVIDER) + .addConfigField(METER_PROVIDER) + .addConfigFieldResolver(RESOLVE_TRACER_PROVIDER) + .addConfigFieldResolver(RESOLVE_METER_PROVIDER) + .build() + ); + } + + @Override + public void writeAdditionalFiles(GoCodegenContext ctx) { + ctx.writerDelegator().useFileWriter("api_client.go", ctx.settings().getModuleName(), goTemplate(""" + func resolveTracerProvider(options *Options) { + if options.TracerProvider == nil { + options.TracerProvider = &$T{} + } + } + + func resolveMeterProvider(options *Options) { + if options.MeterProvider == nil { + options.MeterProvider = $T{} + } + } + """, + SmithyGoDependency.SMITHY_TRACING.struct("NopTracerProvider"), + SmithyGoDependency.SMITHY_METRICS.struct("NopMeterProvider") + )); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/OperationMetricsStruct.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/OperationMetricsStruct.java new file mode 100644 index 000000000..04ad9a30b --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/OperationMetricsStruct.java @@ -0,0 +1,187 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package software.amazon.smithy.go.codegen.integration; + +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.CONTEXT; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_METRICS; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_MIDDLEWARE; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.TIME; + +import software.amazon.smithy.go.codegen.GoWriter; + +/** + * Writable operationMetrics structure that records operation-specific metrics. + */ +public class OperationMetricsStruct implements GoWriter.Writable { + private final String scope; + + public OperationMetricsStruct(String scope) { + this.scope = scope; + } + + @Override + public void accept(GoWriter writer) { + writer.write(GoWriter.ChainWritable.of( + useDependencies(), generateStruct(), generateHelpers(), generateContextApis() + ).compose()); + } + + private GoWriter.Writable useDependencies() { + return writer -> writer + .addUseImports(CONTEXT) + .addUseImports(TIME) + .addUseImports(SMITHY_METRICS) + .addUseImports(SMITHY_MIDDLEWARE); + } + + private GoWriter.Writable generateStruct() { + return goTemplate(""" + type operationMetrics struct { + Duration metrics.Float64Histogram + SerializeDuration metrics.Float64Histogram + ResolveIdentityDuration metrics.Float64Histogram + ResolveEndpointDuration metrics.Float64Histogram + SignRequestDuration metrics.Float64Histogram + DeserializeDuration metrics.Float64Histogram + } + """); + } + + @SuppressWarnings({"checkstyle:LineLength"}) + private GoWriter.Writable generateContextApis() { + return goTemplate(""" + type operationMetricsKey struct{} + + func withOperationMetrics(parent context.Context, mp metrics.MeterProvider) (context.Context, error) { + meter := mp.Meter($S) + om := &operationMetrics{} + + var err error + + om.Duration, err = operationMetricTimer(meter, "client.call.duration", + "Overall call duration (including retries and time to send or receive request and response body)") + if err != nil { + return nil, err + } + om.SerializeDuration, err = operationMetricTimer(meter, "client.call.serialization_duration", + "The time it takes to serialize a message body") + if err != nil { + return nil, err + } + om.ResolveIdentityDuration, err = operationMetricTimer(meter, "client.call.auth.resolve_identity_duration", + "The time taken to acquire an identity (AWS credentials, bearer token, etc) from an Identity Provider") + if err != nil { + return nil, err + } + om.ResolveEndpointDuration, err = operationMetricTimer(meter, "client.call.resolve_endpoint_duration", + "The time it takes to resolve an endpoint (endpoint resolver, not DNS) for the request") + if err != nil { + return nil, err + } + om.SignRequestDuration, err = operationMetricTimer(meter, "client.call.auth.signing_duration", + "The time it takes to sign a request") + if err != nil { + return nil, err + } + om.DeserializeDuration, err = operationMetricTimer(meter, "client.call.deserialization_duration", + "The time it takes to deserialize a message body") + if err != nil { + return nil, err + } + + return context.WithValue(parent, operationMetricsKey{}, om), nil + } + + func operationMetricTimer(m metrics.Meter, name, desc string) (metrics.Float64Histogram, error) { + return m.Float64Histogram(name, func(o *metrics.InstrumentOptions) { + o.UnitLabel = "s" + o.Description = desc + }) + } + + func getOperationMetrics(ctx context.Context) *operationMetrics { + return ctx.Value(operationMetricsKey{}).(*operationMetrics) + } + """, scope); + } + + private GoWriter.Writable generateHelpers() { + return goTemplate(""" + func (m *operationMetrics) histogramFor(name string) metrics.Float64Histogram { + switch name { + case "client.call.duration": + return m.Duration + case "client.call.serialization_duration": + return m.SerializeDuration + case "client.call.resolve_identity_duration": + return m.ResolveIdentityDuration + case "client.call.resolve_endpoint_duration": + return m.ResolveEndpointDuration + case "client.call.signing_duration": + return m.SignRequestDuration + case "client.call.deserialization_duration": + return m.DeserializeDuration + default: + panic("unrecognized operation metric") + } + } + + func timeOperationMetric[T any]( + ctx context.Context, metric string, fn func() (T, error), + opts ...metrics.RecordMetricOption, + ) (T, error) { + instr := getOperationMetrics(ctx).histogramFor(metric) + opts = append([]metrics.RecordMetricOption{withOperationMetadata(ctx)}, opts...) + + start := time.Now() + v, err := fn() + end := time.Now() + + elapsed := end.Sub(start) + instr.Record(ctx, float64(elapsed)/1e9, opts...) + return v, err + } + + func startMetricTimer(ctx context.Context, metric string, opts ...metrics.RecordMetricOption) func() { + instr := getOperationMetrics(ctx).histogramFor(metric) + opts = append([]metrics.RecordMetricOption{withOperationMetadata(ctx)}, opts...) + + var ended bool + start := time.Now() + return func() { + if ended { + return + } + ended = true + + end := time.Now() + + elapsed := end.Sub(start) + instr.Record(ctx, float64(elapsed)/1e9, opts...) + } + } + + func withOperationMetadata(ctx context.Context) metrics.RecordMetricOption { + return func(o *metrics.RecordMetricOptions) { + o.Properties.Set("rpc.service", middleware.GetServiceID(ctx)) + o.Properties.Set("rpc.method", middleware.GetOperationName(ctx)) + } + } + """); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/TracingSpans.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/TracingSpans.java new file mode 100644 index 000000000..2c32b3f25 --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/TracingSpans.java @@ -0,0 +1,166 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + * + */ + +package software.amazon.smithy.go.codegen.integration; + +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_MIDDLEWARE; +import static software.amazon.smithy.go.codegen.SmithyGoDependency.SMITHY_TRACING; +import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol; + +import java.util.List; +import java.util.Map; +import software.amazon.smithy.go.codegen.GoCodegenContext; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.middleware.BuildStepMiddleware; +import software.amazon.smithy.go.codegen.middleware.InitializeStepMiddleware; +import software.amazon.smithy.go.codegen.middleware.SerializeStepMiddleware; + +/** + * Instruments the client with various base trace spans. + */ +public class TracingSpans implements GoIntegration { + public static final MiddlewareRegistrar SPAN_INITIALIZE_START = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addSpanInitializeStart")) + .build(); + public static final MiddlewareRegistrar SPAN_INITIALIZE_END = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addSpanInitializeEnd")) + .build(); + public static final MiddlewareRegistrar SPAN_BUILD_REQUEST_START = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addSpanBuildRequestStart")) + .build(); + public static final MiddlewareRegistrar SPAN_BUILD_REQUEST_END = MiddlewareRegistrar.builder() + .resolvedFunction(buildPackageSymbol("addSpanBuildRequestEnd")) + .build(); + + @Override + public byte getOrder() { + return 127; + } + + @Override + public List getClientPlugins() { + return List.of( + RuntimeClientPlugin.builder().registerMiddleware(SPAN_INITIALIZE_START).build(), + RuntimeClientPlugin.builder().registerMiddleware(SPAN_INITIALIZE_END).build(), + RuntimeClientPlugin.builder().registerMiddleware(SPAN_BUILD_REQUEST_START).build(), + RuntimeClientPlugin.builder().registerMiddleware(SPAN_BUILD_REQUEST_END).build() + ); + } + + @Override + public void writeAdditionalFiles(GoCodegenContext ctx) { + ctx.writerDelegator().useFileWriter("api_client.go", ctx.settings().getModuleName(), goTemplate(""" + $initializeStart:W + $initializeEnd:W + + $buildRequestStart:W + $buildRequestEnd:W + + func addSpanInitializeStart(stack $stack:P) error { + return stack.Initialize.Add(&spanInitializeStart{}, $before:T) + } + + func addSpanInitializeEnd(stack $stack:P) error { + return stack.Initialize.Add(&spanInitializeEnd{}, $after:T) + } + + func addSpanBuildRequestStart(stack $stack:P) error { + return stack.Serialize.Add(&spanBuildRequestStart{}, $before:T) + } + + func addSpanBuildRequestEnd(stack $stack:P) error { + return stack.Build.Add(&spanBuildRequestEnd{}, $after:T) + } + """, + Map.of( + "initializeStart", new SpanInitializeStart(), + "initializeEnd", new SpanInitializeEnd(), + "buildRequestStart", new SpanBuildRequestStart(), + "buildRequestEnd", new SpanBuildRequestEnd(), + "stack", SMITHY_MIDDLEWARE.struct("Stack"), + "before", SMITHY_MIDDLEWARE.constSymbol("Before"), + "after", SMITHY_MIDDLEWARE.constSymbol("After") + ))); + } + + private static final class SpanInitializeStart extends InitializeStepMiddleware { + @Override + public String getStructName() { + return "spanInitializeStart"; + } + + @Override + public GoWriter.Writable getFuncBody() { + return goTemplate(""" + ctx, _ = $T(ctx, "Initialize") + + return next.HandleInitialize(ctx, in) + """, SMITHY_TRACING.func("StartSpan")); + } + } + + private static final class SpanInitializeEnd extends InitializeStepMiddleware { + @Override + public String getStructName() { + return "spanInitializeEnd"; + } + + @Override + public GoWriter.Writable getFuncBody() { + return goTemplate(""" + ctx, span := $T(ctx) + span.End() + + return next.HandleInitialize(ctx, in) + """, SMITHY_TRACING.func("PopSpan")); + } + } + + private static final class SpanBuildRequestStart extends SerializeStepMiddleware { + @Override + public String getStructName() { + return "spanBuildRequestStart"; + } + + @Override + public GoWriter.Writable getFuncBody() { + return goTemplate(""" + ctx, _ = $T(ctx, "BuildRequest") + + return next.HandleSerialize(ctx, in) + """, SMITHY_TRACING.func("StartSpan")); + } + } + + private static final class SpanBuildRequestEnd extends BuildStepMiddleware { + @Override + public String getStructName() { + return "spanBuildRequestEnd"; + } + + @Override + public GoWriter.Writable getFuncBody() { + return goTemplate(""" + ctx, span := $T(ctx) + span.End() + + return next.HandleBuild(ctx, in) + """, SMITHY_TRACING.func("PopSpan")); + } + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/BuildStepMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/BuildStepMiddleware.java new file mode 100644 index 000000000..c348f1195 --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/BuildStepMiddleware.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Abstract class for BuildStep middleware generation. + */ +public abstract class BuildStepMiddleware extends OperationMiddleware { + @Override + public String getFuncName() { + return "HandleBuild"; + } + + @Override + public Symbol getInput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("BuildInput"); + } + + @Override + public Symbol getHandler() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("BuildHandler"); + } + + @Override + public Symbol getOutput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("BuildOutput"); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/DeserializeStepMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/DeserializeStepMiddleware.java new file mode 100644 index 000000000..ab3c916d7 --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/DeserializeStepMiddleware.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Abstract class for DeserializeStep middleware generation. + */ +public abstract class DeserializeStepMiddleware extends OperationMiddleware { + @Override + public String getFuncName() { + return "HandleDeserialize"; + } + + @Override + public Symbol getInput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("DeserializeInput"); + } + + @Override + public Symbol getHandler() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("DeserializeHandler"); + } + + @Override + public Symbol getOutput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("DeserializeOutput"); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/FinalizeStepMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/FinalizeStepMiddleware.java new file mode 100644 index 000000000..cc62de1cb --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/FinalizeStepMiddleware.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Abstract class for FinalizeStep middleware generation. + */ +public abstract class FinalizeStepMiddleware extends OperationMiddleware { + @Override + public String getFuncName() { + return "HandleFinalize"; + } + + @Override + public Symbol getInput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("FinalizeInput"); + } + + @Override + public Symbol getHandler() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("FinalizeHandler"); + } + + @Override + public Symbol getOutput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("FinalizeOutput"); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/InitializeStepMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/InitializeStepMiddleware.java new file mode 100644 index 000000000..1bc316337 --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/InitializeStepMiddleware.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Abstract class for InitializeStep middleware generation. + */ +public abstract class InitializeStepMiddleware extends OperationMiddleware { + @Override + public String getFuncName() { + return "HandleInitialize"; + } + + @Override + public Symbol getInput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("InitializeInput"); + } + + @Override + public Symbol getHandler() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("InitializeHandler"); + } + + @Override + public Symbol getOutput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("InitializeOutput"); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/OperationMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/OperationMiddleware.java new file mode 100644 index 000000000..b9c389d52 --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/OperationMiddleware.java @@ -0,0 +1,91 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import static java.util.Collections.emptyMap; +import static software.amazon.smithy.go.codegen.GoWriter.goTemplate; + +import java.util.Map; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.GoStdlibTypes; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.SmithyGoTypes; + +/** + * Abstract base class for code generation of operation middleware. + */ +public abstract class OperationMiddleware implements GoWriter.Writable { + public abstract String getStructName(); + + public Map getFields() { + return emptyMap(); + } + + public String getId() { + return getStructName(); + } + + public abstract String getFuncName(); + + public abstract Symbol getInput(); + + public abstract Symbol getHandler(); + + public abstract Symbol getOutput(); + + public abstract GoWriter.Writable getFuncBody(); + + @Override + public final void accept(GoWriter goWriter) { + goWriter.write(goTemplate(""" + type $name:L struct { + $fields:W + } + + func (*$name:L) ID() string { + return $id:S + } + + func (m *$name:L) $func:L ( + ctx $context:T, in $in:T, next $next:T, + ) ( + $out:T, $md:T, error, + ) { + $body:W + } + """, + Map.of( + "name", getStructName(), + "fields", renderFields(), + "id", getId(), + "func", getFuncName(), + "context", GoStdlibTypes.Context.Context, + "in", getInput(), + "next", getHandler(), + "out", getOutput(), + "md", SmithyGoTypes.Middleware.Metadata, + "body", getFuncBody() + ))); + } + + private GoWriter.Writable renderFields() { + return GoWriter.ChainWritable.of( + getFields().entrySet().stream() + .map(it -> goTemplate("$L $P", it.getKey(), it.getValue())) + .toList() + ).compose(false); + } +} diff --git a/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/SerializeStepMiddleware.java b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/SerializeStepMiddleware.java new file mode 100644 index 000000000..e7eac87bf --- /dev/null +++ b/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/middleware/SerializeStepMiddleware.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.go.codegen.middleware; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.go.codegen.SmithyGoDependency; + +/** + * Abstract class for SerializeStep middleware generation. + */ +public abstract class SerializeStepMiddleware extends OperationMiddleware { + @Override + public String getFuncName() { + return "HandleSerialize"; + } + + @Override + public Symbol getInput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("SerializeInput"); + } + + @Override + public Symbol getHandler() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("SerializeHandler"); + } + + @Override + public Symbol getOutput() { + return SmithyGoDependency.SMITHY_MIDDLEWARE.struct("SerializeOutput"); + } +} diff --git a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 04ed5213c..cc1cae8e0 100644 --- a/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -13,5 +13,8 @@ software.amazon.smithy.go.codegen.integration.auth.AnonymousAuthScheme software.amazon.smithy.go.codegen.requestcompression.RequestCompression +software.amazon.smithy.go.codegen.integration.ObservabilityOptions +software.amazon.smithy.go.codegen.integration.TracingSpans + # server software.amazon.smithy.go.codegen.server.integration.DefaultProtocols diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 000000000..c009d9f27 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,136 @@ +// Package metrics defines the metrics APIs used by Smithy clients. +package metrics + +import ( + "context" + + "github.com/aws/smithy-go" +) + +// MeterProvider is the entry point for creating a Meter. +type MeterProvider interface { + Meter(scope string, opts ...MeterOption) Meter +} + +// MeterOption applies configuration to a Meter. +type MeterOption func(o *MeterOptions) + +// MeterOptions represents configuration for a Meter. +type MeterOptions struct { + Properties smithy.Properties +} + +// Meter is the entry point for creation of measurement instruments. +type Meter interface { + // integer/synchronous + Int64Counter(name string, opts ...InstrumentOption) (Int64Counter, error) + Int64UpDownCounter(name string, opts ...InstrumentOption) (Int64UpDownCounter, error) + Int64Gauge(name string, opts ...InstrumentOption) (Int64Gauge, error) + Int64Histogram(name string, opts ...InstrumentOption) (Int64Histogram, error) + + // integer/asynchronous + Int64AsyncCounter(name string, callback Int64Callback, opts ...InstrumentOption) (AsyncInstrument, error) + Int64AsyncUpDownCounter(name string, callback Int64Callback, opts ...InstrumentOption) (AsyncInstrument, error) + Int64AsyncGauge(name string, callback Int64Callback, opts ...InstrumentOption) (AsyncInstrument, error) + + // floating-point/synchronous + Float64Counter(name string, opts ...InstrumentOption) (Float64Counter, error) + Float64UpDownCounter(name string, opts ...InstrumentOption) (Float64UpDownCounter, error) + Float64Gauge(name string, opts ...InstrumentOption) (Float64Gauge, error) + Float64Histogram(name string, opts ...InstrumentOption) (Float64Histogram, error) + + // floating-point/asynchronous + Float64AsyncCounter(name string, callback Float64Callback, opts ...InstrumentOption) (AsyncInstrument, error) + Float64AsyncUpDownCounter(name string, callback Float64Callback, opts ...InstrumentOption) (AsyncInstrument, error) + Float64AsyncGauge(name string, callback Float64Callback, opts ...InstrumentOption) (AsyncInstrument, error) +} + +// InstrumentOption applies configuration to an instrument. +type InstrumentOption func(o *InstrumentOptions) + +// InstrumentOptions represents configuration for an instrument. +type InstrumentOptions struct { + UnitLabel string + Description string +} + +// Int64Counter measures a monotonically increasing int64 value. +type Int64Counter interface { + Add(context.Context, int64, ...RecordMetricOption) +} + +// Int64UpDownCounter measures a fluctuating int64 value. +type Int64UpDownCounter interface { + Add(context.Context, int64, ...RecordMetricOption) +} + +// Int64Gauge samples a discrete int64 value. +type Int64Gauge interface { + Sample(context.Context, int64, ...RecordMetricOption) +} + +// Int64Histogram records multiple data points for an int64 value. +type Int64Histogram interface { + Record(context.Context, int64, ...RecordMetricOption) +} + +// Float64Counter measures a monotonically increasing float64 value. +type Float64Counter interface { + Add(context.Context, float64, ...RecordMetricOption) +} + +// Float64UpDownCounter measures a fluctuating float64 value. +type Float64UpDownCounter interface { + Add(context.Context, float64, ...RecordMetricOption) +} + +// Float64Gauge samples a discrete float64 value. +type Float64Gauge interface { + Sample(context.Context, float64, ...RecordMetricOption) +} + +// Float64Histogram records multiple data points for an float64 value. +type Float64Histogram interface { + Record(context.Context, float64, ...RecordMetricOption) +} + +// AsyncInstrument is the universal handle returned for creation of all async +// instruments. +// +// Callers use the Stop() API to unregister the callback passed at instrument +// creation. +type AsyncInstrument interface { + Stop() +} + +// Int64Callback describes a function invoked when an async int64 instrument is +// read. +type Int64Callback func(context.Context, Int64Observer) + +// Int64Observer is the interface passed to async int64 instruments. +// +// Callers use the Observe() API of this interface to report metrics to the +// underlying collector. +type Int64Observer interface { + Observe(context.Context, int64, ...RecordMetricOption) +} + +// Float64Callback describes a function invoked when an async float64 +// instrument is read. +type Float64Callback func(context.Context, Float64Observer) + +// Float64Observer is the interface passed to async int64 instruments. +// +// Callers use the Observe() API of this interface to report metrics to the +// underlying collector. +type Float64Observer interface { + Observe(context.Context, float64, ...RecordMetricOption) +} + +// RecordMetricOption applies configuration to a recorded metric. +type RecordMetricOption func(o *RecordMetricOptions) + +// RecordMetricOptions represents configuration for a recorded metric. +type RecordMetricOptions struct { + Properties smithy.Properties +} diff --git a/metrics/nop.go b/metrics/nop.go new file mode 100644 index 000000000..fb374e1fb --- /dev/null +++ b/metrics/nop.go @@ -0,0 +1,67 @@ +package metrics + +import "context" + +// NopMeterProvider is a no-op metrics implementation. +type NopMeterProvider struct{} + +var _ MeterProvider = (*NopMeterProvider)(nil) + +// Meter returns a meter which creates no-op instruments. +func (NopMeterProvider) Meter(string, ...MeterOption) Meter { + return nopMeter{} +} + +type nopMeter struct{} + +var _ Meter = (*nopMeter)(nil) + +func (nopMeter) Int64Counter(string, ...InstrumentOption) (Int64Counter, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64UpDownCounter(string, ...InstrumentOption) (Int64UpDownCounter, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64Gauge(string, ...InstrumentOption) (Int64Gauge, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64Histogram(string, ...InstrumentOption) (Int64Histogram, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64AsyncCounter(string, Int64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64AsyncUpDownCounter(string, Int64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Int64AsyncGauge(string, Int64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[int64]{}, nil +} +func (nopMeter) Float64Counter(string, ...InstrumentOption) (Float64Counter, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64UpDownCounter(string, ...InstrumentOption) (Float64UpDownCounter, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64Gauge(string, ...InstrumentOption) (Float64Gauge, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64Histogram(string, ...InstrumentOption) (Float64Histogram, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64AsyncCounter(string, Float64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64AsyncUpDownCounter(string, Float64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[float64]{}, nil +} +func (nopMeter) Float64AsyncGauge(string, Float64Callback, ...InstrumentOption) (AsyncInstrument, error) { + return nopInstrument[float64]{}, nil +} + +type nopInstrument[N any] struct{} + +func (nopInstrument[N]) Add(context.Context, N, ...RecordMetricOption) {} +func (nopInstrument[N]) Sample(context.Context, N, ...RecordMetricOption) {} +func (nopInstrument[N]) Record(context.Context, N, ...RecordMetricOption) {} +func (nopInstrument[_]) Stop() {} diff --git a/metrics/smithyotelmetrics/async.go b/metrics/smithyotelmetrics/async.go new file mode 100644 index 000000000..3cb819b82 --- /dev/null +++ b/metrics/smithyotelmetrics/async.go @@ -0,0 +1,62 @@ +package smithyotelmetrics + +import ( + "context" + + "github.com/aws/smithy-go/metrics" + otelmetric "go.opentelemetry.io/otel/metric" +) + +type asyncInstrument struct { + otel otelmetric.Registration +} + +var _ metrics.AsyncInstrument = (*asyncInstrument)(nil) + +func (i *asyncInstrument) Stop() { + i.otel.Unregister() +} + +// int64Observer wraps an untyped, multi-instrument OTEL Observer to Observe() +// against a single int64 instrument. +type int64Observer struct { + observer otelmetric.Observer + instrument otelmetric.Int64Observable +} + +var _ metrics.Int64Observer = (*int64Observer)(nil) + +func (o *int64Observer) Observe(_ context.Context, v int64, opts ...metrics.RecordMetricOption) { + o.observer.ObserveInt64(o.instrument, v, withMetricProps(opts...)) +} + +// adaptInt64CB wraps an OTEL async instrument callback, binding it to a single +// int64 instrument. +func adaptInt64CB(io otelmetric.Int64Observable, cb metrics.Int64Callback) otelmetric.Callback { + return func(ctx context.Context, o otelmetric.Observer) error { + cb(ctx, &int64Observer{o, io}) + return nil + } +} + +// float64Observer wraps an untyped, multi-instrument OTEL Observer to Observe() +// against a single float64 instrument. +type float64Observer struct { + observer otelmetric.Observer + instrument otelmetric.Float64Observable +} + +var _ metrics.Float64Observer = (*float64Observer)(nil) + +func (o *float64Observer) Observe(_ context.Context, v float64, opts ...metrics.RecordMetricOption) { + o.observer.ObserveFloat64(o.instrument, v, withMetricProps(opts...)) +} + +// adaptFloat64CB wraps an OTEL async instrument callback, binding it to a single +// float64 instrument. +func adaptFloat64CB(io otelmetric.Float64Observable, cb metrics.Float64Callback) otelmetric.Callback { + return func(ctx context.Context, o otelmetric.Observer) error { + cb(ctx, &float64Observer{o, io}) + return nil + } +} diff --git a/metrics/smithyotelmetrics/attribute.go b/metrics/smithyotelmetrics/attribute.go new file mode 100644 index 000000000..abacc40e6 --- /dev/null +++ b/metrics/smithyotelmetrics/attribute.go @@ -0,0 +1,67 @@ +package smithyotelmetrics + +import ( + "fmt" + + "github.com/aws/smithy-go" + otelattribute "go.opentelemetry.io/otel/attribute" +) + +// IMPORTANT: The contents of this file are mirrored in +// smithyoteltracing/attribute.go. Any changes made here must be replicated in +// that module's copy of the file, although that will probably never happen, as +// the set of attribute types supported by the OTEL API cannot reasonably +// expand to include anything else that would be useful. +// +// This is done in order to avoid the one-way door of exposing an internal-only +// module for what is effectively a simple value mapper (that will likely never +// change). +// +// While the contents of the file are mirrored, the tests are only present +// in the other version. + +func toOTELKeyValue(k, v any) otelattribute.KeyValue { + kk := str(k) + + switch vv := v.(type) { + case bool: + return otelattribute.Bool(kk, vv) + case []bool: + return otelattribute.BoolSlice(kk, vv) + case int: + return otelattribute.Int(kk, vv) + case []int: + return otelattribute.IntSlice(kk, vv) + case int64: + return otelattribute.Int64(kk, vv) + case []int64: + return otelattribute.Int64Slice(kk, vv) + case float64: + return otelattribute.Float64(kk, vv) + case []float64: + return otelattribute.Float64Slice(kk, vv) + case string: + return otelattribute.String(kk, vv) + case []string: + return otelattribute.StringSlice(kk, vv) + default: + return otelattribute.String(kk, str(v)) + } +} + +func toOTELKeyValues(props smithy.Properties) []otelattribute.KeyValue { + var kvs []otelattribute.KeyValue + for k, v := range props.Values() { + kvs = append(kvs, toOTELKeyValue(k, v)) + } + return kvs +} + +func str(v any) string { + if s, ok := v.(string); ok { + return s + } else if s, ok := v.(fmt.Stringer); ok { + return s.String() + } + return fmt.Sprintf("%#v", v) +} diff --git a/metrics/smithyotelmetrics/doc.go b/metrics/smithyotelmetrics/doc.go new file mode 100644 index 000000000..6025b5101 --- /dev/null +++ b/metrics/smithyotelmetrics/doc.go @@ -0,0 +1,40 @@ +// Package smithyotelmetrics implements a Smithy client metrics adapter for the +// OTEL Go SDK. +// +// # Usage +// +// Callers use the [Adapt] API in this package to wrap a concrete OTEL SDK +// MeterProvider. +// +// The following example uses the AWS SDK for S3: +// +// import ( +// "github.com/aws/aws-sdk-go-v2/config" +// "github.com/aws/aws-sdk-go-v2/service/s3" +// "github.com/aws/smithy-go/metrics/smithyotelmetrics" +// "go.opentelemetry.io/otel/sdk/metric" +// "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" +// ) +// +// func main() { +// cfg, err := config.LoadDefaultConfig(context.Background()) +// if err != nil { +// panic(err) +// } +// +// // export via OTLP - perhaps to the otel collector, etc. +// exporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithEndpointURL("http://localhost:4318")) +// if err != nil { +// panic(err) +// } +// +// // aggressive reader interval for demonstration purposes +// reader := metric.NewPeriodicReader(exporter, metric.WithInterval(time.Second)) +// provider := metric.NewMeterProvider(metric.WithReader(reader) +// +// svc := s3.NewFromConfig(cfg, func(o *s3.Options) { +// o.MeterProvider = smithyotelmetrics.Adapt(provider) +// }) +// // ... +// } +package smithyotelmetrics diff --git a/metrics/smithyotelmetrics/float64.go b/metrics/smithyotelmetrics/float64.go new file mode 100644 index 000000000..696d69674 --- /dev/null +++ b/metrics/smithyotelmetrics/float64.go @@ -0,0 +1,43 @@ +package smithyotelmetrics + +import ( + "context" + + "github.com/aws/smithy-go/metrics" + otelmetric "go.opentelemetry.io/otel/metric" +) + +type otelFloat64Add interface { + Add(context.Context, float64, ...otelmetric.AddOption) +} + +type float64Counter struct { + otel otelFloat64Add +} + +var _ metrics.Float64Counter = (*float64Counter)(nil) +var _ metrics.Float64UpDownCounter = (*float64Counter)(nil) + +func (i *float64Counter) Add(ctx context.Context, v float64, opts ...metrics.RecordMetricOption) { + i.otel.Add(ctx, v, withMetricProps(opts...)) +} + +type float64Gauge struct { + otel otelmetric.Float64Gauge +} + +var _ metrics.Float64Gauge = (*float64Gauge)(nil) + +func (i *float64Gauge) Sample(ctx context.Context, v float64, opts ...metrics.RecordMetricOption) { + i.otel.Record(ctx, v, withMetricProps(opts...)) +} + +type float64Histogram struct { + otel otelmetric.Float64Histogram +} + +var _ metrics.Float64Histogram = (*float64Histogram)(nil) + +func (i *float64Histogram) Record(ctx context.Context, v float64, opts ...metrics.RecordMetricOption) { + i.otel.Record(ctx, v, withMetricProps(opts...)) +} diff --git a/metrics/smithyotelmetrics/go.mod b/metrics/smithyotelmetrics/go.mod new file mode 100644 index 000000000..ba9e618eb --- /dev/null +++ b/metrics/smithyotelmetrics/go.mod @@ -0,0 +1,11 @@ +module github.com/aws/smithy-go/metrics/smithyotelmetrics + +go 1.22 + +require ( + github.com/aws/smithy-go v1.20.4 + go.opentelemetry.io/otel v1.29.0 + go.opentelemetry.io/otel/metric v1.29.0 +) + +replace github.com/aws/smithy-go => ../../ diff --git a/metrics/smithyotelmetrics/go.sum b/metrics/smithyotelmetrics/go.sum new file mode 100644 index 000000000..6296a714c --- /dev/null +++ b/metrics/smithyotelmetrics/go.sum @@ -0,0 +1,20 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/smithyotelmetrics/int64.go b/metrics/smithyotelmetrics/int64.go new file mode 100644 index 000000000..ca0783126 --- /dev/null +++ b/metrics/smithyotelmetrics/int64.go @@ -0,0 +1,41 @@ +package smithyotelmetrics + +import ( + "context" + + "github.com/aws/smithy-go/metrics" + otelmetric "go.opentelemetry.io/otel/metric" +) + +type int64Counter struct { + otel interface { + Add(context.Context, int64, ...otelmetric.AddOption) + } +} + +var _ metrics.Int64Counter = (*int64Counter)(nil) +var _ metrics.Int64UpDownCounter = (*int64Counter)(nil) + +func (i *int64Counter) Add(ctx context.Context, v int64, opts ...metrics.RecordMetricOption) { + i.otel.Add(ctx, v, withMetricProps(opts...)) +} + +type int64Gauge struct { + otel otelmetric.Int64Gauge +} + +var _ metrics.Int64Gauge = (*int64Gauge)(nil) + +func (i *int64Gauge) Sample(ctx context.Context, v int64, opts ...metrics.RecordMetricOption) { + i.otel.Record(ctx, v, withMetricProps(opts...)) +} + +type int64Histogram struct { + otel otelmetric.Int64Histogram +} + +var _ metrics.Int64Histogram = (*int64Histogram)(nil) + +func (i *int64Histogram) Record(ctx context.Context, v int64, opts ...metrics.RecordMetricOption) { + i.otel.Record(ctx, v, withMetricProps(opts...)) +} diff --git a/metrics/smithyotelmetrics/metrics.go b/metrics/smithyotelmetrics/metrics.go new file mode 100644 index 000000000..650bc5a2f --- /dev/null +++ b/metrics/smithyotelmetrics/metrics.go @@ -0,0 +1,188 @@ +package smithyotelmetrics + +import ( + "github.com/aws/smithy-go/metrics" + otelmetric "go.opentelemetry.io/otel/metric" +) + +// Adapt wraps a concrete OpenTelemetry SDK MeterProvider for use with Smithy +// SDK clients. +// +// Adapt can be called multiple times on a single MeterProvider. +func Adapt(mp otelmetric.MeterProvider) metrics.MeterProvider { + return &meterProvider{mp} +} + +type meterProvider struct { + otel otelmetric.MeterProvider +} + +var _ metrics.MeterProvider = (*meterProvider)(nil) + +func (p *meterProvider) Meter(scope string, opts ...metrics.MeterOption) metrics.Meter { + var options metrics.MeterOptions + for _, opt := range opts { + opt(&options) + } + + m := p.otel.Meter(scope, otelmetric.WithInstrumentationAttributes( + toOTELKeyValues(options.Properties)..., + )) + return &meter{m} +} + +type meter struct { + otel otelmetric.Meter +} + +var _ metrics.Meter = (*meter)(nil) + +func (m *meter) Int64Counter(name string, opts ...metrics.InstrumentOption) (metrics.Int64Counter, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64Counter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &int64Counter{i}, nil +} + +func (m *meter) Int64UpDownCounter(name string, opts ...metrics.InstrumentOption) (metrics.Int64UpDownCounter, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64UpDownCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &int64Counter{i}, nil +} + +func (m *meter) Int64Gauge(name string, opts ...metrics.InstrumentOption) (metrics.Int64Gauge, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64Gauge(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &int64Gauge{i}, nil +} + +func (m *meter) Int64Histogram(name string, opts ...metrics.InstrumentOption) (metrics.Int64Histogram, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64Histogram(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &int64Histogram{i}, nil +} + +func (m *meter) Int64AsyncCounter(name string, callback metrics.Int64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64ObservableCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncInt64(i, callback) +} + +func (m *meter) Int64AsyncUpDownCounter(name string, callback metrics.Int64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64ObservableUpDownCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncInt64(i, callback) +} + +func (m *meter) Int64AsyncGauge(name string, callback metrics.Int64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Int64ObservableGauge(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncInt64(i, callback) +} + +func (m *meter) Float64Counter(name string, opts ...metrics.InstrumentOption) (metrics.Float64Counter, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64Counter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &float64Counter{i}, nil +} + +func (m *meter) Float64UpDownCounter(name string, opts ...metrics.InstrumentOption) (metrics.Float64UpDownCounter, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64UpDownCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &float64Counter{i}, nil +} + +func (m *meter) Float64Gauge(name string, opts ...metrics.InstrumentOption) (metrics.Float64Gauge, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64Gauge(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &float64Gauge{i}, nil +} + +func (m *meter) Float64Histogram(name string, opts ...metrics.InstrumentOption) (metrics.Float64Histogram, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64Histogram(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + return &float64Histogram{i}, nil +} + +func (m *meter) Float64AsyncCounter(name string, callback metrics.Float64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64ObservableCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncFloat64(i, callback) +} + +func (m *meter) Float64AsyncUpDownCounter(name string, callback metrics.Float64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64ObservableUpDownCounter(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncFloat64(i, callback) +} + +func (m *meter) Float64AsyncGauge(name string, callback metrics.Float64Callback, opts ...metrics.InstrumentOption) (metrics.AsyncInstrument, error) { + unit, desc := toInstrumentOpts(opts...) + i, err := m.otel.Float64ObservableGauge(name, otelmetric.WithUnit(unit), otelmetric.WithDescription(desc)) + if err != nil { + return nil, err + } + + return m.registerAsyncFloat64(i, callback) +} + +func (m *meter) registerAsyncInt64(i otelmetric.Int64Observable, cb metrics.Int64Callback) (metrics.AsyncInstrument, error) { + r, err := m.otel.RegisterCallback(adaptInt64CB(i, cb), i) + if err != nil { + return nil, err + } + + return &asyncInstrument{r}, nil +} + +func (m *meter) registerAsyncFloat64(i otelmetric.Float64Observable, cb metrics.Float64Callback) (metrics.AsyncInstrument, error) { + r, err := m.otel.RegisterCallback(adaptFloat64CB(i, cb), i) + if err != nil { + return nil, err + } + + return &asyncInstrument{r}, nil +} diff --git a/metrics/smithyotelmetrics/option.go b/metrics/smithyotelmetrics/option.go new file mode 100644 index 000000000..2ec977dde --- /dev/null +++ b/metrics/smithyotelmetrics/option.go @@ -0,0 +1,23 @@ +package smithyotelmetrics + +import ( + "github.com/aws/smithy-go/metrics" + otelmetric "go.opentelemetry.io/otel/metric" +) + +func toInstrumentOpts(opts ...metrics.InstrumentOption) (unit, desc string) { + var o metrics.InstrumentOptions + for _, opt := range opts { + opt(&o) + } + return o.UnitLabel, o.Description +} + +func withMetricProps(opts ...metrics.RecordMetricOption) otelmetric.MeasurementOption { + var o metrics.RecordMetricOptions + for _, opt := range opts { + opt(&o) + } + return otelmetric.WithAttributes(toOTELKeyValues(o.Properties)...) + +} diff --git a/middleware/context.go b/middleware/context.go new file mode 100644 index 000000000..f51aa4f04 --- /dev/null +++ b/middleware/context.go @@ -0,0 +1,41 @@ +package middleware + +import "context" + +type ( + serviceIDKey struct{} + operationNameKey struct{} +) + +// WithServiceID adds a service ID to the context, scoped to middleware stack +// values. +// +// This API is called in the client runtime when bootstrapping an operation and +// should not typically be used directly. +func WithServiceID(parent context.Context, id string) context.Context { + return WithStackValue(parent, serviceIDKey{}, id) +} + +// GetServiceID retrieves the service ID from the context. This is typically +// the service shape's name from its Smithy model. Service clients for specific +// systems (e.g. AWS SDK) may use an alternate designated value. +func GetServiceID(ctx context.Context) string { + id, _ := GetStackValue(ctx, serviceIDKey{}).(string) + return id +} + +// WithOperationName adds the operation name to the context, scoped to +// middleware stack values. +// +// This API is called in the client runtime when bootstrapping an operation and +// should not typically be used directly. +func WithOperationName(parent context.Context, id string) context.Context { + return WithStackValue(parent, operationNameKey{}, id) +} + +// GetOperationName retrieves the operation name from the context. This is +// typically the operation shape's name from its Smithy model. +func GetOperationName(ctx context.Context) string { + name, _ := GetStackValue(ctx, operationNameKey{}).(string) + return name +} diff --git a/properties.go b/properties.go index c9af66c0e..68df4c4e0 100644 --- a/properties.go +++ b/properties.go @@ -1,9 +1,11 @@ package smithy +import "maps" + // PropertiesReader provides an interface for reading metadata from the // underlying metadata container. type PropertiesReader interface { - Get(key interface{}) interface{} + Get(key any) any } // Properties provides storing and reading metadata values. Keys may be any @@ -12,14 +14,14 @@ type PropertiesReader interface { // The zero value for a Properties instance is ready for reads/writes without // any additional initialization. type Properties struct { - values map[interface{}]interface{} + values map[any]any } // Get attempts to retrieve the value the key points to. Returns nil if the // key was not found. // // Panics if key type is not comparable. -func (m *Properties) Get(key interface{}) interface{} { +func (m *Properties) Get(key any) any { m.lazyInit() return m.values[key] } @@ -28,7 +30,7 @@ func (m *Properties) Get(key interface{}) interface{} { // that key it will be replaced with the new value. // // Panics if the key type is not comparable. -func (m *Properties) Set(key, value interface{}) { +func (m *Properties) Set(key, value any) { m.lazyInit() m.values[key] = value } @@ -36,7 +38,7 @@ func (m *Properties) Set(key, value interface{}) { // Has returns whether the key exists in the metadata. // // Panics if the key type is not comparable. -func (m *Properties) Has(key interface{}) bool { +func (m *Properties) Has(key any) bool { m.lazyInit() _, ok := m.values[key] return ok @@ -55,8 +57,13 @@ func (m *Properties) SetAll(other *Properties) { } } +// Values returns a shallow clone of the property set's values. +func (m *Properties) Values() map[any]any { + return maps.Clone(m.values) +} + func (m *Properties) lazyInit() { if m.values == nil { - m.values = map[interface{}]interface{}{} + m.values = map[any]any{} } } diff --git a/tracing/context.go b/tracing/context.go new file mode 100644 index 000000000..a404ed9d3 --- /dev/null +++ b/tracing/context.go @@ -0,0 +1,96 @@ +package tracing + +import "context" + +type ( + operationTracerKey struct{} + spanLineageKey struct{} +) + +// GetSpan returns the active trace Span on the context. +// +// The boolean in the return indicates whether a Span was actually in the +// context, but a no-op implementation will be returned if not, so callers +// can generally disregard the boolean unless they wish to explicitly confirm +// presence/absence of a Span. +func GetSpan(ctx context.Context) (Span, bool) { + lineage := getLineage(ctx) + if len(lineage) == 0 { + return nopSpan{}, false + } + + return lineage[len(lineage)-1], true +} + +// WithSpan sets the active trace Span on the context. +func WithSpan(parent context.Context, span Span) context.Context { + lineage := getLineage(parent) + if len(lineage) == 0 { + return context.WithValue(parent, spanLineageKey{}, []Span{span}) + } + + lineage = append(lineage, span) + return context.WithValue(parent, spanLineageKey{}, lineage) +} + +// PopSpan pops the current Span off the context, setting the active Span on +// the returned Context back to its parent and returning the REMOVED one. +// +// PopSpan on a context with no active Span will return a no-op instance. +// +// This is mostly necessary for the runtime to manage base trace spans due to +// the wrapped-function nature of the middleware stack. End-users of Smithy +// clients SHOULD NOT generally be using this API. +func PopSpan(parent context.Context) (context.Context, Span) { + lineage := getLineage(parent) + if len(lineage) == 0 { + return parent, nopSpan{} + } + + span := lineage[len(lineage)-1] + lineage = lineage[:len(lineage)-1] + return context.WithValue(parent, spanLineageKey{}, lineage), span +} + +func getLineage(ctx context.Context) []Span { + v := ctx.Value(spanLineageKey{}) + if v == nil { + return nil + } + + return v.([]Span) +} + +// GetOperationTracer returns the embedded operation-scoped Tracer on a +// Context. +// +// The boolean in the return indicates whether a Tracer was actually in the +// context, but a no-op implementation will be returned if not, so callers +// can generally disregard the boolean unless they wish to explicitly confirm +// presence/absence of a Tracer. +func GetOperationTracer(ctx context.Context) (Tracer, bool) { + v := ctx.Value(operationTracerKey{}) + if v == nil { + return nopTracer{}, false + } + + return v.(Tracer), true +} + +// WithOperationTracer returns a child Context embedding the given Tracer. +// +// The runtime will use this embed a scoped tracer for client operations, +// Smithy/SDK client callers DO NOT need to do this explicitly. +func WithOperationTracer(parent context.Context, tracer Tracer) context.Context { + return context.WithValue(parent, operationTracerKey{}, tracer) +} + +// StartSpan is a convenience API for creating tracing Spans from a Context. +// +// StartSpan uses the operation-scoped Tracer, previously stored using +// [WithOperationTracer], to start the Span. If a Tracer has not been embedded +// the returned Span will be a no-op implementation. +func StartSpan(ctx context.Context, name string, opts ...SpanOption) (context.Context, Span) { + tracer, _ := GetOperationTracer(ctx) + return tracer.StartSpan(ctx, name, opts...) +} diff --git a/tracing/context_test.go b/tracing/context_test.go new file mode 100644 index 000000000..a6d0cb34c --- /dev/null +++ b/tracing/context_test.go @@ -0,0 +1,64 @@ +package tracing + +import ( + "context" + "testing" +) + +// nopSpan has no fields so all values are equal, adding an int allows us to +// differentiate +type mockSpan struct { + nopSpan + id int +} + +func TestSpanContextAPIs(t *testing.T) { + parent := mockSpan{id: 1} + child := mockSpan{id: 2} + + ctx := context.Background() + span, ok := GetSpan(ctx) + if ok { + t.Error("should have no span at the start but it did") + } + + // set & expect parent + ctx = WithSpan(ctx, parent) + span, _ = GetSpan(ctx) + if actual, ok := span.(mockSpan); !ok || parent != actual { + t.Errorf("span %d != %d", parent.id, actual.id) + } + + // set & expect child + ctx = WithSpan(ctx, child) + span, _ = GetSpan(ctx) + if actual, ok := span.(mockSpan); !ok || child != actual { + t.Errorf("span %d != %d", child.id, actual.id) + } + + // pop, expect popped child, with parent remaining + ctx, span = PopSpan(ctx) + if actual, ok := span.(mockSpan); !ok || child != actual { + t.Errorf("span %d != %d", child.id, actual.id) + } + span, _ = GetSpan(ctx) + if actual, ok := span.(mockSpan); !ok || parent != actual { + t.Errorf("span %d != %d", parent.id, actual.id) + } + + // pop, expect popped parent, with no span remaining + ctx, span = PopSpan(ctx) + if actual, ok := span.(mockSpan); !ok || parent != actual { + t.Errorf("span %d != %d", parent.id, actual.id) + } + span, ok = GetSpan(ctx) + if ok { + t.Error("should have no span at the end but it did") + } + + // pop, expect it to be a nop since nothing is left + ctx, span = PopSpan(ctx) + if _, ok := span.(nopSpan); !ok { + t.Errorf("should have been nop span on last pop but was %T", span) + } +} diff --git a/tracing/nop.go b/tracing/nop.go new file mode 100644 index 000000000..573d28b1c --- /dev/null +++ b/tracing/nop.go @@ -0,0 +1,32 @@ +package tracing + +import "context" + +// NopTracerProvider is a no-op tracing implementation. +type NopTracerProvider struct{} + +var _ TracerProvider = (*NopTracerProvider)(nil) + +// Tracer returns a tracer which creates no-op spans. +func (NopTracerProvider) Tracer(string, ...TracerOption) Tracer { + return nopTracer{} +} + +type nopTracer struct{} + +var _ Tracer = (*nopTracer)(nil) + +func (nopTracer) StartSpan(ctx context.Context, name string, opts ...SpanOption) (context.Context, Span) { + return ctx, nopSpan{} +} + +type nopSpan struct{} + +var _ Span = (*nopSpan)(nil) + +func (nopSpan) Name() string { return "" } +func (nopSpan) Context() SpanContext { return SpanContext{} } +func (nopSpan) AddEvent(string, ...EventOption) {} +func (nopSpan) SetProperty(any, any) {} +func (nopSpan) SetStatus(SpanStatus) {} +func (nopSpan) End() {} diff --git a/tracing/smithyoteltracing/adapt.go b/tracing/smithyoteltracing/adapt.go new file mode 100644 index 000000000..9058f0167 --- /dev/null +++ b/tracing/smithyoteltracing/adapt.go @@ -0,0 +1,140 @@ +package smithyoteltracing + +import ( + "context" + + "github.com/aws/smithy-go/tracing" + otelcodes "go.opentelemetry.io/otel/codes" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// Adapt wraps a concrete OpenTelemetry SDK TraceProvider for use with Smithy +// SDK clients. +// +// Adapt can be called multiple times on a single TracerProvider. +func Adapt(tp oteltrace.TracerProvider) tracing.TracerProvider { + return &tracerProvider{tp} +} + +type tracerProvider struct { + otel oteltrace.TracerProvider +} + +var _ tracing.TracerProvider = (*tracerProvider)(nil) + +func (p *tracerProvider) Tracer(scope string, opts ...tracing.TracerOption) tracing.Tracer { + var options tracing.TracerOptions + for _, opt := range opts { + opt(&options) + } + + t := p.otel.Tracer(scope, oteltrace.WithInstrumentationAttributes( + toOTELKeyValues(options.Properties)..., + )) + return &tracer{t} +} + +type tracer struct { + otel oteltrace.Tracer +} + +var _ tracing.Tracer = (*tracer)(nil) + +func (t *tracer) StartSpan(ctx context.Context, name string, opts ...tracing.SpanOption) (context.Context, tracing.Span) { + // We do some context value juggling with our adapted Span to ensure the + // following: + // (1) Our adapted Span is what actually persists on the context and + // is what callers are getting and recording to. + // (2) OTEL itself sees any pre-existing Span such that the parent-child + // relationship of concrete OTEL spans is maintained. + ours, ok := tracing.GetSpan(ctx) + if ok { + ctx = oteltrace.ContextWithSpan(ctx, ours.(*span).otel) // (2) + } + + var options tracing.SpanOptions + for _, opt := range opts { + opt(&options) + } + + kind := toOTELSpanKind(options.Kind) + ctx, theirs := t.otel.Start(ctx, name, oteltrace.WithSpanKind(kind)) + + ours = &span{ + otel: theirs, + name: name, + } + for k, v := range options.Properties.Values() { + ours.SetProperty(k, v) + } + return tracing.WithSpan(ctx, ours) /* (1) */, ours +} + +type span struct { + otel oteltrace.Span + name string +} + +var _ tracing.Span = (*span)(nil) + +func (s *span) Name() string { + return s.name +} + +func (s *span) Context() tracing.SpanContext { + ctx := s.otel.SpanContext() + return tracing.SpanContext{ + TraceID: ctx.TraceID().String(), + SpanID: ctx.SpanID().String(), + IsRemote: ctx.IsRemote(), + } +} + +func (s *span) AddEvent(name string, opts ...tracing.EventOption) { + var options tracing.EventOptions + for _, opt := range opts { + opt(&options) + } + + s.otel.AddEvent(name, oteltrace.WithAttributes( + toOTELKeyValues(options.Properties)..., + )) +} + +func (s *span) SetProperty(k, v any) { + s.otel.SetAttributes(toOTELKeyValue(k, v)) +} + +func (s *span) SetStatus(status tracing.SpanStatus) { + s.otel.SetStatus(toOTELSpanStatus(status), "") +} + +func (s *span) End() { + s.otel.End() +} + +func toOTELSpanKind(v tracing.SpanKind) oteltrace.SpanKind { + switch v { + case tracing.SpanKindClient: + return oteltrace.SpanKindClient + case tracing.SpanKindServer: + return oteltrace.SpanKindServer + case tracing.SpanKindProducer: + return oteltrace.SpanKindProducer + case tracing.SpanKindConsumer: + return oteltrace.SpanKindConsumer + default: + return oteltrace.SpanKindInternal + } +} + +func toOTELSpanStatus(v tracing.SpanStatus) otelcodes.Code { + switch v { + case tracing.SpanStatusOK: + return otelcodes.Ok + case tracing.SpanStatusError: + return otelcodes.Error + default: + return otelcodes.Unset + } +} diff --git a/tracing/smithyoteltracing/adapt_test.go b/tracing/smithyoteltracing/adapt_test.go new file mode 100644 index 000000000..293806d1e --- /dev/null +++ b/tracing/smithyoteltracing/adapt_test.go @@ -0,0 +1,52 @@ +package smithyoteltracing + +import ( + "fmt" + "testing" + + "github.com/aws/smithy-go/tracing" + otelcodes "go.opentelemetry.io/otel/codes" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func TestToOTELSpanKind(t *testing.T) { + for _, tt := range []struct { + In tracing.SpanKind + Expect oteltrace.SpanKind + }{ + {tracing.SpanKindClient, oteltrace.SpanKindClient}, + {tracing.SpanKindServer, oteltrace.SpanKindServer}, + {tracing.SpanKindProducer, oteltrace.SpanKindProducer}, + {tracing.SpanKindConsumer, oteltrace.SpanKindConsumer}, + {tracing.SpanKindInternal, oteltrace.SpanKindInternal}, + {tracing.SpanKind(-1), oteltrace.SpanKindInternal}, + } { + name := fmt.Sprintf("%v -> %v", tt.In, tt.Expect) + t.Run(name, func(t *testing.T) { + actual := toOTELSpanKind(tt.In) + if tt.Expect != actual { + t.Errorf("%v != %v", tt.Expect, actual) + } + }) + } +} + +func TestToOTELSpanStatus(t *testing.T) { + for _, tt := range []struct { + In tracing.SpanStatus + Expect otelcodes.Code + }{ + {tracing.SpanStatusOK, otelcodes.Ok}, + {tracing.SpanStatusError, otelcodes.Error}, + {tracing.SpanStatusUnset, otelcodes.Unset}, + {tracing.SpanStatus(-1), otelcodes.Unset}, + } { + name := fmt.Sprintf("%v -> %v", tt.In, tt.Expect) + t.Run(name, func(t *testing.T) { + actual := toOTELSpanStatus(tt.In) + if tt.Expect != actual { + t.Errorf("%v != %v", tt.Expect, actual) + } + }) + } +} diff --git a/tracing/smithyoteltracing/attribute.go b/tracing/smithyoteltracing/attribute.go new file mode 100644 index 000000000..b51b81a56 --- /dev/null +++ b/tracing/smithyoteltracing/attribute.go @@ -0,0 +1,67 @@ +package smithyoteltracing + +import ( + "fmt" + + "github.com/aws/smithy-go" + otelattribute "go.opentelemetry.io/otel/attribute" +) + +// IMPORTANT: The contents of this file are mirrored in +// smithyotelmetrics/attribute.go. Any changes made here must be replicated in +// that module's copy of the file, although that will probably never happen, as +// the set of attribute types supported by the OTEL API cannot reasonably +// expand to include anything else that would be useful. +// +// This is done in order to avoid the one-way door of exposing an internal-only +// module for what is effectively a simple value mapper (that will likely never +// change). +// +// While the contents of the file are mirrored, the tests are only present +// here. + +func toOTELKeyValue(k, v any) otelattribute.KeyValue { + kk := str(k) + + switch vv := v.(type) { + case bool: + return otelattribute.Bool(kk, vv) + case []bool: + return otelattribute.BoolSlice(kk, vv) + case int: + return otelattribute.Int(kk, vv) + case []int: + return otelattribute.IntSlice(kk, vv) + case int64: + return otelattribute.Int64(kk, vv) + case []int64: + return otelattribute.Int64Slice(kk, vv) + case float64: + return otelattribute.Float64(kk, vv) + case []float64: + return otelattribute.Float64Slice(kk, vv) + case string: + return otelattribute.String(kk, vv) + case []string: + return otelattribute.StringSlice(kk, vv) + default: + return otelattribute.String(kk, str(v)) + } +} + +func toOTELKeyValues(props smithy.Properties) []otelattribute.KeyValue { + var kvs []otelattribute.KeyValue + for k, v := range props.Values() { + kvs = append(kvs, toOTELKeyValue(k, v)) + } + return kvs +} + +func str(v any) string { + if s, ok := v.(string); ok { + return s + } else if s, ok := v.(fmt.Stringer); ok { + return s.String() + } + return fmt.Sprintf("%#v", v) +} diff --git a/tracing/smithyoteltracing/attribute_test.go b/tracing/smithyoteltracing/attribute_test.go new file mode 100644 index 000000000..0b36abbc3 --- /dev/null +++ b/tracing/smithyoteltracing/attribute_test.go @@ -0,0 +1,46 @@ +package smithyoteltracing + +import ( + "fmt" + "testing" + + otelattribute "go.opentelemetry.io/otel/attribute" +) + +type stringer struct{} + +func (s stringer) String() string { + return "stringer" +} + +type notstringer struct{} + +func TestToOTELKeyValue(t *testing.T) { + for _, tt := range []struct { + K, V any + Expect otelattribute.KeyValue + }{ + {1, "asdf", otelattribute.String("1", "asdf")}, // non-string key + {"key", stringer{}, otelattribute.String("key", "stringer")}, // stringer + // unsupported value type + {"key", notstringer{}, otelattribute.String("key", "smithyoteltracing.notstringer{}")}, + {"key", true, otelattribute.Bool("key", true)}, + {"key", []bool{true, false}, otelattribute.BoolSlice("key", []bool{true, false})}, + {"key", int(1), otelattribute.Int("key", 1)}, + {"key", []int{1, 2}, otelattribute.IntSlice("key", []int{1, 2})}, + {"key", int64(1), otelattribute.Int64("key", 1)}, + {"key", []int64{1, 2}, otelattribute.Int64Slice("key", []int64{1, 2})}, + {"key", float64(1), otelattribute.Float64("key", 1)}, + {"key", []float64{1, 2}, otelattribute.Float64Slice("key", []float64{1, 2})}, + {"key", "value", otelattribute.String("key", "value")}, + {"key", []string{"v1", "v2"}, otelattribute.StringSlice("key", []string{"v1", "v2"})}, + } { + name := fmt.Sprintf("(%v, %v) -> %v", tt.K, tt.V, tt.Expect) + t.Run(name, func(t *testing.T) { + actual := toOTELKeyValue(tt.K, tt.V) + if tt.Expect != actual { + t.Errorf("%v != %v", tt.Expect, actual) + } + }) + } +} diff --git a/tracing/smithyoteltracing/doc.go b/tracing/smithyoteltracing/doc.go new file mode 100644 index 000000000..a22b69893 --- /dev/null +++ b/tracing/smithyoteltracing/doc.go @@ -0,0 +1,69 @@ +// Package smithyoteltracing implements a Smithy client tracing adapter for the +// OTEL Go SDK. +// +// # Usage +// +// Callers use the [Adapt] API in this package to wrap a concrete OTEL SDK +// TracerProvider. +// +// The following example uses the AWS SDK for S3: +// +// import ( +// "github.com/aws/aws-sdk-go-v2/config" +// "github.com/aws/aws-sdk-go-v2/service/s3" +// "github.com/aws/smithy-go/tracing/smithyoteltracing" +// "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" +// "go.opentelemetry.io/otel/sdk/trace" +// ) +// +// func main() { +// exporter, err := stdouttrace.New() +// if err != nil { +// panic(err) +// } +// +// cfg, err := config.LoadDefaultConfig(context.Background()) +// if err != nil { +// panic(err) +// } +// +// provider := trace.NewTracerProvider(trace.WithBatcher(exporter)) +// svc := s3.NewFromConfig(cfg, func(o *s3.Options) { +// o.TracerProvider = smithyoteltracing.Adapt(provider) +// }) +// // ... +// } +// +// # OTEL Attributes +// +// This adapter supports all attribute types used in the OTEL SDK (including +// their slice-of variants): +// - bool +// - int +// - int64 +// - float64 +// - string +// +// A key-value pair set on a [smithy.Properties] container in any of the +// tracing APIs will automatically propagate to the underlying OTEL SDK if its +// key is of type string and its value is one of those supported. Otherwise, +// the adapter will make an effort to stringify the key or value as follows: +// - if value implements String(), use that value +// - otherwise, use the Go-string representation of the value (the result of fmt.Sprintf("%#v", ...)) +// +// e.g. +// +// type key struct{} +// +// func (k key) String() string { +// return "app.key" +// } +// +// type key2 struct{} +// +// ctx, span := tracing.StartSpan(ctx, "Foo", func(o *tracing.SpanOptions) { +// o.Properties.Set("app.version", "bar") // -> ("app.version", "bar") +// o.Properties.Set(key{}, "baz") // -> ("app.key", "baz") +// o.Properties.Set(key2{}, "qux") // -> ("main.key2{}", "qux") +// }) +package smithyoteltracing diff --git a/tracing/smithyoteltracing/go.mod b/tracing/smithyoteltracing/go.mod new file mode 100644 index 000000000..bf2f4016d --- /dev/null +++ b/tracing/smithyoteltracing/go.mod @@ -0,0 +1,11 @@ +module github.com/aws/smithy-go/tracing/smithyoteltracing + +go 1.22 + +require ( + github.com/aws/smithy-go v1.20.4 + go.opentelemetry.io/otel v1.29.0 + go.opentelemetry.io/otel/trace v1.29.0 +) + +replace github.com/aws/smithy-go => ../../ diff --git a/tracing/smithyoteltracing/go.sum b/tracing/smithyoteltracing/go.sum new file mode 100644 index 000000000..117dd580e --- /dev/null +++ b/tracing/smithyoteltracing/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tracing/tracing.go b/tracing/tracing.go new file mode 100644 index 000000000..089ed3932 --- /dev/null +++ b/tracing/tracing.go @@ -0,0 +1,95 @@ +// Package tracing defines tracing APIs to be used by Smithy clients. +package tracing + +import ( + "context" + + "github.com/aws/smithy-go" +) + +// SpanStatus records the "success" state of an observed span. +type SpanStatus int + +// Enumeration of SpanStatus. +const ( + SpanStatusUnset SpanStatus = iota + SpanStatusOK + SpanStatusError +) + +// SpanKind indicates the nature of the work being performed. +type SpanKind int + +// Enumeration of SpanKind. +const ( + SpanKindInternal SpanKind = iota + SpanKindClient + SpanKindServer + SpanKindProducer + SpanKindConsumer +) + +// TracerProvider is the entry point for creating client traces. +type TracerProvider interface { + Tracer(scope string, opts ...TracerOption) Tracer +} + +// TracerOption applies configuration to a tracer. +type TracerOption func(o *TracerOptions) + +// TracerOptions represent configuration for tracers. +type TracerOptions struct { + Properties smithy.Properties +} + +// Tracer is the entry point for creating observed client Spans. +// +// Spans created by tracers propagate by existing on the Context. Consumers of +// the API can use [GetSpan] to pull the active Span from a Context. +// +// Creation of child Spans is implicit through Context persistence. If +// CreateSpan is called with a Context that holds a Span, the result will be a +// child of that Span. +type Tracer interface { + StartSpan(ctx context.Context, name string, opts ...SpanOption) (context.Context, Span) +} + +// SpanOption applies configuration to a span. +type SpanOption func(o *SpanOptions) + +// SpanOptions represent configuration for span events. +type SpanOptions struct { + Kind SpanKind + Properties smithy.Properties +} + +// Span records a conceptually individual unit of work that takes place in a +// Smithy client operation. +type Span interface { + Name() string + Context() SpanContext + AddEvent(name string, opts ...EventOption) + SetStatus(status SpanStatus) + SetProperty(k, v any) + End() +} + +// EventOption applies configuration to a span event. +type EventOption func(o *EventOptions) + +// EventOptions represent configuration for span events. +type EventOptions struct { + Properties smithy.Properties +} + +// SpanContext uniquely identifies a Span. +type SpanContext struct { + TraceID string + SpanID string + IsRemote bool +} + +// IsValid is true when a span has nonzero trace and span IDs. +func (ctx *SpanContext) IsValid() bool { + return len(ctx.TraceID) != 0 && len(ctx.SpanID) != 0 +} diff --git a/transport/http/client.go b/transport/http/client.go index e691c69bf..c43c346b6 100644 --- a/transport/http/client.go +++ b/transport/http/client.go @@ -7,6 +7,7 @@ import ( smithy "github.com/aws/smithy-go" "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/tracing" ) // ClientDo provides the interface for custom HTTP client implementations. @@ -42,6 +43,9 @@ func NewClientHandler(client ClientDo) ClientHandler { func (c ClientHandler) Handle(ctx context.Context, input interface{}) ( out interface{}, metadata middleware.Metadata, err error, ) { + ctx, span := tracing.StartSpan(ctx, "DoHTTPRequest") + defer span.End() + req, ok := input.(*Request) if !ok { return nil, metadata, fmt.Errorf("expect Smithy http.Request value as input, got unsupported type %T", input) @@ -52,6 +56,16 @@ func (c ClientHandler) Handle(ctx context.Context, input interface{}) ( return nil, metadata, err } + span.SetProperty("http.method", req.Method) + span.SetProperty("http.request_content_length", -1) // at least indicate unknown + length, ok, err := req.StreamLength() + if err != nil { + return nil, metadata, err + } + if ok { + span.SetProperty("http.request_content_length", length) + } + resp, err := c.client.Do(builtRequest) if resp == nil { // Ensure a http response value is always present to prevent unexpected @@ -79,6 +93,10 @@ func (c ClientHandler) Handle(ctx context.Context, input interface{}) ( _ = builtRequest.Body.Close() } + span.SetProperty("net.protocol.version", fmt.Sprintf("%d.%d", resp.ProtoMajor, resp.ProtoMinor)) + span.SetProperty("http.status_code", resp.StatusCode) + span.SetProperty("http.response_content_length", resp.ContentLength) + return &Response{Response: resp}, metadata, err }