Multi Error Reporting, Custom Report Attrs, and Clean Up

Implemented ability to report multiple apm errors from one log. If a user adds
multiple "reportable" error attributes to the log msg (default is "error" & "err"),
instead of trying to join the errors into one or discarding one, the apmslog
handler will report both errors.

Added ability for a user to define what slog attribute keys they want to report
as errors. Because there is no standard way in slog to attach an error to a msg
log, I wanted to add the ability for the user to decide what is and what is not
going to be reported. By default, slog attribute keys that are "error" or "err"
are reported, but with the new `WithErrorRecordAttrs(keys)` function a user
can define which keys will be reported.

Cleaned up `ApmHandler` struct and methods. Since we want the user to use the
included `NewApmHandler` function and its functional option functions, I
decided to make all Struct fields private.

Additionally added a check on if the `ApmHandler`'s `tracer` field is nill before
trying to use it. It is still possible for a user to pass in a nil tracer using
the `WithTracer` functional option.

New tests and documentation added.

docs/instrumenting.asciidoc

@@ -814,7 +814,17 @@ func ExampleHandler() {
 	span, ctx := apm.StartSpan(ctx, "name", "type")
 	defer span.End()
 
+	// log msg will have a trace, transaction, and a span attached
 	logger.InfoContext(ctx, "I should have a trace, transaction, and span id attached!")
+
+	// the log msg will be reported to apm
+	logger.ErrorContext(ctx, "I want this to be reported, but have no error to attach")
+
+	// the log msg with its error will be reported to apm
+	logger.ErrorContext(ctx, "I will report this error to apm", "error", errors.New("new error"))
+
+	// BOTH errors with the log msg will be reported to apm. [ error, err ] slog attribute keys are by default reported
+	logger.ErrorContext(ctx, "I will report this error to apm", "error", errors.New("new error"), "err", errors.New("new err"))
 }
 ----
 

module/apmslog/example_test.go

@@ -19,6 +19,7 @@ package apmslog_test
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 
 	"go.elastic.co/apm/module/apmslog/v2"
@@ -41,5 +42,15 @@ func ExampleHandler() {
 	span, ctx := apm.StartSpan(ctx, "name", "type")
 	defer span.End()
 
+	// log msg will have a trace, transaction, and a span attached
 	logger.InfoContext(ctx, "I should have a trace, transaction, and span id attached!")
+
+	// the log msg will be reported to apm
+	logger.ErrorContext(ctx, "I want this to be reported, but have no error to attach")
+
+	// the log msg with its error will be reported to apm
+	logger.ErrorContext(ctx, "I will report this error to apm", "error", errors.New("new error"))
+
+	// BOTH errors with the log msg will be reported to apm. [ error, err ] slog attribute keys are by default reported
+	logger.ErrorContext(ctx, "I will report this error to apm", "error", errors.New("new error"), "err", errors.New("new err"))
 }

module/apmslog/go.mod

@@ -3,7 +3,7 @@ module go.elastic.co/apm/module/apmslog/v2
 require (
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4
-	go.elastic.co/apm/v2 v2.5.0
+	go.elastic.co/apm/v2 v2.6.0
 )
 
 require (
@@ -23,4 +23,4 @@ require (
 
 replace go.elastic.co/apm/v2 => ../..
 
-go 1.19
+go 1.21

module/apmslog/go.sum

@@ -53,6 +53,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

module/apmslog/handler.go

@@ -45,35 +45,28 @@ const (
 )
 
 type ApmHandler struct {
-	Tracer       *apm.Tracer
-	ReportLevels []slog.Level
-	Handler      slog.Handler
-}
-
-func (h *ApmHandler) tracer() *apm.Tracer {
-	if h.Tracer == nil {
-		return apm.DefaultTracer()
-	}
-	return h.Tracer
+	tracer           *apm.Tracer
+	reportLevels     []slog.Level
+	errorRecordAttrs []string
+	handler          slog.Handler
 }
 
 // Enabled reports whether the handler handles records at the given level.
 func (h *ApmHandler) Enabled(ctx context.Context, level slog.Level) bool {
-	return h.Handler.Enabled(ctx, level)
+	return h.handler.Enabled(ctx, level)
 }
 
 // WithAttrs returns a new ApmHandler with passed attributes attached.
 func (h *ApmHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
-	return &ApmHandler{h.Tracer, h.ReportLevels, h.Handler.WithAttrs(attrs)}
+	return &ApmHandler{h.tracer, h.reportLevels, h.errorRecordAttrs, h.handler.WithAttrs(attrs)}
 }
 
 // WithGroup returns a new ApmHandler with passed group attached.
 func (h *ApmHandler) WithGroup(name string) slog.Handler {
-	return &ApmHandler{h.Tracer, h.ReportLevels, h.Handler.WithGroup(name)}
+	return &ApmHandler{h.tracer, h.reportLevels, h.errorRecordAttrs, h.handler.WithGroup(name)}
 }
 
 func (h *ApmHandler) Handle(ctx context.Context, r slog.Record) error {
-
 	// attempt to extract any available trace info from context
 	var traceId apm.TraceID
 	var transactionId apm.SpanID
@@ -93,60 +86,83 @@ func (h *ApmHandler) Handle(ctx context.Context, r slog.Record) error {
 	}
 
 	// report record as APM error
-	tracer := h.tracer()
-	if slices.Contains(h.ReportLevels, r.Level) && tracer.Recording() {
+	if h.tracer != nil && h.tracer.Recording() && slices.Contains(h.reportLevels, r.Level) {
 
-		// attempt to find error/err attribute
+		// attempt to find error attributes
 		// slog doesnt have a standard way of attaching an
 		// error to a record, so attempting to grab any attribute
-		// that has error/err as key and extracting the value
-		// seems like a likely way to do it.
-		var err error
+		// that has error/err keys OR keys user has defined as reportable
+		// and extracting the values seems like a likely way to do it.
+		errorsToAttach := []error{}
 		r.Attrs(func(a slog.Attr) bool {
-			if a.Key == SlogErrorKeyErr || a.Key == SlogErrorKeyError {
+			if slices.Contains(h.errorRecordAttrs, a.Key) {
+				var err error
+				// first check if value is of error type to retain as much info as possible
 				if v, ok := a.Value.Any().(error); ok {
-					err = v
-					return false
+					errorsToAttach = append(errorsToAttach, v)
+					// else just convert reportable error value as string
 				} else {
-					err = errors.Join(err, fmt.Errorf("%s", a.Value.String()))
-					return false
+					errorsToAttach = append(errorsToAttach, errors.Join(err, fmt.Errorf("%s", a.Value.String())))
 				}
 			}
 			return true
 		})
 
-		errlog := tracer.NewErrorLog(apm.ErrorLogRecord{
-			Message: r.Message,
-			Level:   strings.ToLower(r.Level.String()),
-			Error:   err,
-		})
-		errlog.Handled = true
-		errlog.Timestamp = r.Time.UTC()
-		errlog.SetStacktrace(2)
-
-		// add available trace info if not zero type
-		if traceId != (apm.TraceID{}) {
-			errlog.TraceID = traceId
-		}
-		if transactionId != (apm.SpanID{}) {
-			errlog.TransactionID = transactionId
+		// If there are multiple reportable error attributes, create a new
+		// apm.ErrorLogRecord for each. Otherwise just create one apm.ErrorLogRecord
+		// with no Error.
+		errLogRecords := []apm.ErrorLogRecord{}
+		if len(errorsToAttach) == 0 {
+			errRecord := apm.ErrorLogRecord{
+				Message: r.Message,
+				Level:   strings.ToLower(r.Level.String()),
+			}
+			errLogRecords = append(errLogRecords, errRecord)
+		} else {
+			for _, err := range errorsToAttach {
+				errRecord := apm.ErrorLogRecord{
+					Message: r.Message,
+					Level:   strings.ToLower(r.Level.String()),
+					Error:   err,
+				}
+				errLogRecords = append(errLogRecords, errRecord)
+			}
 		}
-		if parentId != (apm.SpanID{}) {
-			errlog.ParentID = parentId
+
+		// for each errRecord, send to apm
+		for _, errRecord := range errLogRecords {
+			errlog := h.tracer.NewErrorLog(errRecord)
+			errlog.Handled = true
+			errlog.Timestamp = r.Time.UTC()
+			errlog.SetStacktrace(2)
+
+			// add available trace info if not zero type
+			if traceId != (apm.TraceID{}) {
+				errlog.TraceID = traceId
+			}
+			if transactionId != (apm.SpanID{}) {
+				errlog.TransactionID = transactionId
+			}
+			if parentId != (apm.SpanID{}) {
+				errlog.ParentID = parentId
+			}
+			// send error to APM
+			errlog.Send()
+
 		}
-		// send error to APM
-		errlog.Send()
 	}
 
-	return h.Handler.Handle(ctx, r)
+	return h.handler.Handle(ctx, r)
 }
 
 type apmHandlerOption func(h *ApmHandler)
 
+// Create a new ApmHandler.
 func NewApmHandler(opts ...apmHandlerOption) *ApmHandler {
 	h := &ApmHandler{
 		apm.DefaultTracer(),
 		[]slog.Level{slog.LevelError},
+		[]string{SlogErrorKeyErr, SlogErrorKeyError},
 		slog.Default().Handler(),
 	}
 	for _, opt := range opts {
@@ -155,20 +171,34 @@ func NewApmHandler(opts ...apmHandlerOption) *ApmHandler {
 	return h
 }
 
+// Set slog handler for ApmHandler
+// default: slog.Default().Handler()
 func WithHandler(handler slog.Handler) apmHandlerOption {
 	return func(h *ApmHandler) {
-		h.Handler = handler
+		h.handler = handler
 	}
 }
 
+// Set which slog log level will be reported
+// default: slog.LevelError
 func WithReportLevel(lvls []slog.Level) apmHandlerOption {
 	return func(h *ApmHandler) {
-		h.ReportLevels = lvls
+		h.reportLevels = lvls
+	}
+}
+
+// Set with slog attribute keys will be used as errors.
+// default: 'error','err'
+func WithErrorRecordAttrs(keys []string) apmHandlerOption {
+	return func(h *ApmHandler) {
+		h.errorRecordAttrs = keys
 	}
 }
 
+// Set custom tracer for ApmHandler.
+// default: apm.DefaultTracer()
 func WithTracer(tracer *apm.Tracer) apmHandlerOption {
 	return func(h *ApmHandler) {
-		h.Tracer = tracer
+		h.tracer = tracer
 	}
 }