add tracing and metrics support to generated clients (#538)

.changelog/76820480d9aa4b7389f4b8ca928a1522.json

@@ -0,0 +1,8 @@
+{
+    "id": "76820480-d9aa-4b73-89f4-b8ca928a1522",
+    "type": "feature",
+    "description": "Add tracing and metrics APIs, and builtin instrumentation for both, in generated clients.",
+    "modules": [
+        "."
+    ]
+}

.changelog/8a8ce2aaa0b84978a2b874ca2f90d021.json

@@ -0,0 +1,8 @@
+{
+    "id": "8a8ce2aa-a0b8-4978-a2b8-74ca2f90d021",
+    "type": "release",
+    "description": "Initial release of `smithyotelmetrics` module, which is used to adapt an OpenTelemetry SDK meter provider to be used with Smithy clients.",
+    "modules": [
+        "metrics/smithyotelmetrics"
+    ]
+}

.changelog/93daa8ff4ccc4cb8a8a02aef737bb20c.json

@@ -0,0 +1,8 @@
+{
+    "id": "93daa8ff-4ccc-4cb8-a8a0-2aef737bb20c",
+    "type": "release",
+    "description": "Initial release of `smithyoteltracing` module, which is used to adapt an OpenTelemetry SDK tracer provider to be used with Smithy clients.",
+    "modules": [
+        "tracing/smithyoteltracing"
+    ]
+}

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/GoWriter.java

@@ -59,7 +59,7 @@ public final class GoWriter extends SymbolWriter<GoWriter, ImportDeclarations> {
 
     private static final Logger LOGGER = Logger.getLogger(GoWriter.class.getName());
     private static final int DEFAULT_DOC_WRAP_LENGTH = 80;
-    private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]+)(:\\w)?");
+    private static final Pattern ARGUMENT_NAME_PATTERN = Pattern.compile("\\$([a-z][a-zA-Z_0-9]\\.+)(:\\w)?");
     private final String fullPackageName;
     private final boolean innerWriter;
     private final List<String> buildTags = new ArrayList<>();
@@ -97,6 +97,12 @@ private void init() {
         putFormatter('W', new GoWritableInjector());
         putFormatter('D', new GoDependencyFormatter());
 
+        putContext("fmt.Sprintf", SmithyGoDependency.FMT.func("Sprintf"));
+        putContext("fmt.Errorf", SmithyGoDependency.FMT.func("Errorf"));
+        putContext("errors.As", SmithyGoDependency.ERRORS.func("As"));
+        putContext("context.Context", SmithyGoDependency.CONTEXT.func("Context"));
+        putContext("time.Now", SmithyGoDependency.TIME.func("Now"));
+
         if (!innerWriter) {
             packageDocs = new GoWriter(this.fullPackageName, true);
         }

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/ServiceGenerator.java

@@ -37,6 +37,7 @@
 import software.amazon.smithy.go.codegen.integration.ClientMemberResolver;
 import software.amazon.smithy.go.codegen.integration.ConfigFieldResolver;
 import software.amazon.smithy.go.codegen.integration.GoIntegration;
+import software.amazon.smithy.go.codegen.integration.OperationMetricsStruct;
 import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.knowledge.ServiceIndex;
@@ -94,6 +95,7 @@ public void run() {
     private GoWriter.Writable generate() {
         return GoWriter.ChainWritable.of(
                 generateMetadata(),
+                generateObservabilityComponents(),
                 generateClient(),
                 generateNew(),
                 generateGetOptions(),
@@ -103,6 +105,22 @@ private GoWriter.Writable generate() {
         ).compose();
     }
 
+    private GoWriter.Writable generateObservabilityComponents() {
+        return goTemplate("""
+                $operationMetrics:W
+
+                func operationTracer(p $tracerProvider:T) $tracer:T {
+                    return p.Tracer($scope:S)
+                }
+                """,
+                Map.of(
+                        "tracerProvider", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("TracerProvider"),
+                        "tracer", SmithyGoDependency.SMITHY_TRACING.interfaceSymbol("Tracer"),
+                        "scope", settings.getModuleName(),
+                        "operationMetrics", new OperationMetricsStruct(settings.getModuleName())
+                ));
+    }
+
     private GoWriter.Writable generateMetadata() {
         var serviceId = settings.getService().toString();
         for (var integration : integrations) {
@@ -324,8 +342,16 @@ func resolveAuthSchemes(options *Options) {
     @SuppressWarnings("checkstyle:LineLength")
     private GoWriter.Writable generateInvokeOperation() {
         return goTemplate("""
-                func (c *Client) invokeOperation(ctx $context:T, opID string, params interface{}, optFns []func(*Options), stackFns ...func($stack:P, Options) error) (result interface{}, metadata $metadata:T, err error) {
-                    ctx = $clearStackValues:T(ctx)
+                $middleware:D $tracing:D
+                func (c *Client) invokeOperation(
+                    ctx context.Context, opID string, params interface{}, optFns []func(*Options), stackFns ...func(*middleware.Stack, Options) error,
+                ) (
+                    result interface{}, metadata middleware.Metadata, err error,
+                ) {
+                    ctx = middleware.ClearStackValues(ctx)
+                    ctx = middleware.WithServiceID(ctx, ServiceID)
+                    ctx = middleware.WithOperationName(ctx, opID)
+
                     $newStack:W
                     options := c.options.Copy()
                     $resolvers:W
@@ -348,25 +374,61 @@ private GoWriter.Writable generateInvokeOperation() {
                         }
                     }
 
-                    $newStackHandler:W
-                    result, metadata, err = handler.Handle(ctx, params)
+                    ctx, err = withOperationMetrics(ctx, options.MeterProvider)
                     if err != nil {
+                        return nil, metadata, err
+                    }
+
+                    tracer := operationTracer(options.TracerProvider)
+                    spanName := fmt.Sprintf("%s.%s", ServiceID, opID)
+
+                    ctx = tracing.WithOperationTracer(ctx, tracer)
+
+                    ctx, span := tracer.StartSpan(ctx, spanName, func (o *tracing.SpanOptions) {
+                        o.Kind = tracing.SpanKindClient
+                        o.Properties.Set("rpc.system", "aws-api")
+                        o.Properties.Set("rpc.method", opID)
+                        o.Properties.Set("rpc.service", ServiceID)
+                    })
+                    endTimer := startMetricTimer(ctx, "client.call.duration")
+                    defer endTimer()
+                    defer span.End()
+
+                    handler := $newClientHandler:T(options.HTTPClient)
+                    decorated := middleware.DecorateHandler(handler, stack)
+                    result, metadata, err = decorated.Handle(ctx, params)
+                    if err != nil {
+                        span.SetProperty("exception.type", fmt.Sprintf("%T", err))
+                        span.SetProperty("exception.message", err.Error())
+
+                        var aerr smithy.APIError
+                        if $errors.As:T(err, &aerr) {
+                            span.SetProperty("api.error_code", aerr.ErrorCode())
+                            span.SetProperty("api.error_message", aerr.ErrorMessage())
+                            span.SetProperty("api.error_fault", aerr.ErrorFault().String())
+                        }
+
                         err = &$operationError:T{
                             ServiceID: ServiceID,
                             OperationName: opID,
                             Err: err,
                         }
                     }
+
+                    span.SetProperty("error", err != nil)
+                    if err == nil {
+                        span.SetStatus(tracing.SpanStatusOK)
+                    } else {
+                        span.SetStatus(tracing.SpanStatusError)
+                    }
+
                     return result, metadata, err
                 }
                 """,
                 MapUtils.of(
-                        "context", GoStdlibTypes.Context.Context,
-                        "stack", SmithyGoTypes.Middleware.Stack,
-                        "metadata", SmithyGoTypes.Middleware.Metadata,
-                        "clearStackValues", SmithyGoTypes.Middleware.ClearStackValues,
+                        "middleware", SmithyGoDependency.SMITHY_MIDDLEWARE,
+                        "tracing", SmithyGoDependency.SMITHY_TRACING,
                         "newStack", generateNewStack(),
-                        "newStackHandler", generateNewStackHandler(),
                         "operationError", SmithyGoTypes.Smithy.OperationError,
                         "resolvers", GoWriter.ChainWritable.of(
                                 getConfigResolvers(
@@ -379,7 +441,8 @@ private GoWriter.Writable generateInvokeOperation() {
                                         ConfigFieldResolver.Location.OPERATION,
                                         ConfigFieldResolver.Target.FINALIZATION
                                 ).map(this::generateConfigFieldResolver).toList()
-                        ).compose()
+                        ).compose(),
+                        "newClientHandler", SmithyGoDependency.SMITHY_HTTP_TRANSPORT.func("NewClientHandler")
                 ));
     }
 
@@ -389,12 +452,6 @@ private GoWriter.Writable generateNewStack() {
                 SmithyGoTypes.Middleware.NewStack, SmithyGoTypes.Transport.Http.NewStackRequest);
     }
 
-    private GoWriter.Writable generateNewStackHandler() {
-        ensureSupportedProtocol();
-        return goTemplate("handler := $T($T(options.HTTPClient), stack)",
-                SmithyGoTypes.Middleware.DecorateHandler, SmithyGoTypes.Transport.Http.NewClientHandler);
-    }
-
     private void ensureSupportedProtocol() {
         if (!applicationProtocol.isHttpProtocol()) {
             throw new UnsupportedOperationException(

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java

@@ -76,6 +76,8 @@ public final class SmithyGoDependency {
     public static final GoDependency SMITHY_AUTH_BEARER = smithy("auth/bearer");
     public static final GoDependency SMITHY_ENDPOINTS = smithy("endpoints", "smithyendpoints");
     public static final GoDependency SMITHY_ENDPOINT_RULESFN = smithy("endpoints/private/rulesfn");
+    public static final GoDependency SMITHY_TRACING = smithy("tracing");
+    public static final GoDependency SMITHY_METRICS = smithy("metrics");
 
     public static final GoDependency GO_JMESPATH = goJmespath(null);
     public static final GoDependency MATH = stdlib("math");

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/GetIdentityMiddlewareGenerator.java

@@ -21,6 +21,7 @@
 import software.amazon.smithy.go.codegen.GoStdlibTypes;
 import software.amazon.smithy.go.codegen.GoWriter;
 import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
+import software.amazon.smithy.go.codegen.SmithyGoDependency;
 import software.amazon.smithy.go.codegen.SmithyGoTypes;
 import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
 import software.amazon.smithy.utils.MapUtils;
@@ -64,26 +65,39 @@ private GoWriter.Writable generateFields() {
 
     private GoWriter.Writable generateBody() {
         return goTemplate("""
-                rscheme := getResolvedAuthScheme(ctx)
+                innerCtx, span := $startSpan:T(ctx, "GetIdentity")
+                defer span.End()
+
+                rscheme := getResolvedAuthScheme(innerCtx)
                 if rscheme == nil {
-                    return out, metadata, $errorf:T("no resolved auth scheme")
+                    return out, metadata, $fmt.Errorf:T("no resolved auth scheme")
                 }
 
                 resolver := rscheme.Scheme.IdentityResolver(m.options)
                 if resolver == nil {
-                    return out, metadata, $errorf:T("no identity resolver")
+                    return out, metadata, $fmt.Errorf:T("no identity resolver")
                 }
 
-                identity, err := resolver.GetIdentity(ctx, rscheme.IdentityProperties)
+                identity, err := timeOperationMetric(ctx, "client.call.resolve_identity_duration",
+                    func() ($identity:T, error) {
+                        return resolver.GetIdentity(innerCtx, rscheme.IdentityProperties)
+                    },
+                    func (o $recordMetricOptions:P) {
+                        o.Properties.Set("auth.scheme_id", rscheme.Scheme.SchemeID())
+                    })
                 if err != nil {
-                    return out, metadata, $errorf:T("get identity: %w", err)
+                    return out, metadata, $fmt.Errorf:T("get identity: %w", err)
                 }
 
                 ctx = setIdentity(ctx, identity)
+
+                span.End()
                 return next.HandleFinalize(ctx, in)
                 """,
                 MapUtils.of(
-                        "errorf", GoStdlibTypes.Fmt.Errorf
+                        "startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan"),
+                        "identity", SmithyGoDependency.SMITHY_AUTH.interfaceSymbol("Identity"),
+                        "recordMetricOptions", SmithyGoDependency.SMITHY_METRICS.struct("RecordMetricOptions")
                 ));
     }
 

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/ResolveAuthSchemeMiddlewareGenerator.java

@@ -22,6 +22,7 @@
 import software.amazon.smithy.go.codegen.GoStdlibTypes;
 import software.amazon.smithy.go.codegen.GoWriter;
 import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
+import software.amazon.smithy.go.codegen.SmithyGoDependency;
 import software.amazon.smithy.go.codegen.SmithyGoTypes;
 import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
 import software.amazon.smithy.utils.MapUtils;
@@ -66,6 +67,9 @@ private GoWriter.Writable generateFields() {
 
     private GoWriter.Writable generateBody() {
         return goTemplate("""
+                _, span := $3T(ctx, "ResolveAuthScheme")
+                defer span.End()
+
                 params := $1L(ctx, m.operation, getOperationInput(ctx), m.options)
                 options, err := m.options.AuthSchemeResolver.ResolveAuthSchemes(ctx, params)
                 if err != nil {
@@ -78,10 +82,14 @@ private GoWriter.Writable generateBody() {
                 }
 
                 ctx = setResolvedAuthScheme(ctx, scheme)
+
+                span.SetProperty("auth.scheme_id", scheme.Scheme.SchemeID())
+                span.End()
                 return next.HandleFinalize(ctx, in)
                 """,
                 AuthParametersResolverGenerator.FUNC_NAME,
-                GoStdlibTypes.Fmt.Errorf
+                GoStdlibTypes.Fmt.Errorf,
+                SmithyGoDependency.SMITHY_TRACING.func("StartSpan")
         );
     }
 

codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/auth/SignRequestMiddlewareGenerator.java

@@ -16,12 +16,12 @@
 package software.amazon.smithy.go.codegen.auth;
 
 import static software.amazon.smithy.go.codegen.GoStackStepMiddlewareGenerator.createFinalizeStepMiddleware;
-import static software.amazon.smithy.go.codegen.GoWriter.emptyGoTemplate;
 import static software.amazon.smithy.go.codegen.GoWriter.goTemplate;
 
 import software.amazon.smithy.go.codegen.GoStdlibTypes;
 import software.amazon.smithy.go.codegen.GoWriter;
 import software.amazon.smithy.go.codegen.MiddlewareIdentifier;
+import software.amazon.smithy.go.codegen.SmithyGoDependency;
 import software.amazon.smithy.go.codegen.SmithyGoTypes;
 import software.amazon.smithy.go.codegen.endpoints.EndpointMiddlewareGenerator;
 import software.amazon.smithy.go.codegen.integration.ProtocolGenerator;
@@ -39,7 +39,7 @@ public SignRequestMiddlewareGenerator(ProtocolGenerator.GenerationContext contex
 
     public static GoWriter.Writable generateAddToProtocolFinalizers() {
         return goTemplate("""
-                if err := stack.Finalize.Insert(&$L{}, $S, $T); err != nil {
+                if err := stack.Finalize.Insert(&$L{options: options}, $S, $T); err != nil {
                     return $T("add $L: %w", err)
                 }
                 """,
@@ -56,41 +56,53 @@ public GoWriter.Writable generate() {
     }
 
     private GoWriter.Writable generateFields() {
-        return emptyGoTemplate();
+        return goTemplate("""
+                options Options
+                """);
     }
 
     private GoWriter.Writable generateBody() {
         return goTemplate("""
+                _, span := $startSpan:T(ctx, "SignRequest")
+                defer span.End()
+
                 req, ok := in.Request.($request:P)
                 if !ok {
-                    return out, metadata, $errorf:T("unexpected transport type %T", in.Request)
+                    return out, metadata, $fmt.Errorf:T("unexpected transport type %T", in.Request)
                 }
 
                 rscheme := getResolvedAuthScheme(ctx)
                 if rscheme == nil {
-                    return out, metadata, $errorf:T("no resolved auth scheme")
+                    return out, metadata, $fmt.Errorf:T("no resolved auth scheme")
                 }
 
                 identity := getIdentity(ctx)
                 if identity == nil {
-                    return out, metadata, $errorf:T("no identity")
+                    return out, metadata, $fmt.Errorf:T("no identity")
                 }
 
                 signer := rscheme.Scheme.Signer()
                 if signer == nil {
-                    return out, metadata, $errorf:T("no signer")
+                    return out, metadata, $fmt.Errorf:T("no signer")
                 }
 
-                if err := signer.SignRequest(ctx, req, identity, rscheme.SignerProperties); err != nil {
-                    return out, metadata, $errorf:T("sign request: %w", err)
+                _, err = timeOperationMetric(ctx, "client.call.signing_duration", func() (any, error) {
+                    return nil, signer.SignRequest(ctx, req, identity, rscheme.SignerProperties)
+                }, func(o $recordMetricOptions:P) {
+                    o.Properties.Set("auth.scheme_id", rscheme.Scheme.SchemeID())
+                })
+                if err != nil {
+                    return out, metadata, $fmt.Errorf:T("sign request: %w", err)
                 }
 
+                span.End()
                 return next.HandleFinalize(ctx, in)
                 """,
                 MapUtils.of(
                         // FUTURE(#458) protocol generator should specify the transport type
                         "request", SmithyGoTypes.Transport.Http.Request,
-                        "errorf", GoStdlibTypes.Fmt.Errorf
+                        "startSpan", SmithyGoDependency.SMITHY_TRACING.func("StartSpan"),
+                        "recordMetricOptions", SmithyGoDependency.SMITHY_METRICS.struct("RecordMetricOptions")
                 ));
     }
 }