From a5db348672f2bb659147c486d21a04c78862186b Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 07:39:30 -0700 Subject: [PATCH 01/15] Modify ForceFlush to abort after timeout/cancellation --- CHANGELOG.md | 1 + sdk/trace/batch_span_processor.go | 18 +++++++++++++++- sdk/trace/batch_span_processor_test.go | 30 ++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0080ffa2ebb..2c63578d0dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Jaeger exporter was updated to use thrift v0.14.1. (#1712) - Migrate from using internally built and maintained version of the OTLP to the one hosted at `go.opentelemetry.io/proto/otlp`. (#1713) - Migrate from using `github.com/gogo/protobuf` to `google.golang.org/protobuf` to match `go.opentelemetry.io/proto/otlp`. (#1713) +- Modify `BatchSpanProcessor.ForceFlush` to abort after timeout/cancellation (#1618) ### Removed diff --git a/sdk/trace/batch_span_processor.go b/sdk/trace/batch_span_processor.go index aa322ae6461..616a4c9eb69 100644 --- a/sdk/trace/batch_span_processor.go +++ b/sdk/trace/batch_span_processor.go @@ -148,7 +148,23 @@ func (bsp *batchSpanProcessor) Shutdown(ctx context.Context) error { // ForceFlush exports all ended spans that have not yet been exported. func (bsp *batchSpanProcessor) ForceFlush(ctx context.Context) error { - return bsp.exportSpans(ctx) + var err error + if bsp.e != nil { + wait := make(chan struct{}) + go func() { + if err := bsp.exportSpans(ctx); err != nil { + otel.Handle(err) + } + close(wait) + }() + // Wait until the wait group is done or the context is cancelled/timed out + select { + case <-wait: + case <-ctx.Done(): + err = ctx.Err() + } + } + return err } func WithMaxQueueSize(size int) BatchSpanProcessorOption { diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 6b721ffe322..56946aa3dba 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -257,3 +257,33 @@ func TestBatchSpanProcessorShutdown(t *testing.T) { } assert.Equal(t, 1, bp.shutdownCount) } + +func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { + var bp testBatchExporter + bsp := sdktrace.NewBatchSpanProcessor(&bp) + + err := bsp.ForceFlush(context.Background()) + assert.Equal(t, nil, err) +} + +func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { + var bp testBatchExporter + bsp := sdktrace.NewBatchSpanProcessor(&bp) + // Add timeout to context to test deadline + ctx, cancel := context.WithTimeout(context.Background(), 0) + defer cancel() + + err := bsp.ForceFlush(ctx) + assert.Equal(t, context.DeadlineExceeded, err) +} + +func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) { + var bp testBatchExporter + bsp := sdktrace.NewBatchSpanProcessor(&bp) + ctx, cancel := context.WithCancel(context.Background()) + // Cancel the context + cancel() + + err := bsp.ForceFlush(ctx) + assert.Equal(t, context.Canceled, err) +} \ No newline at end of file From 98ae7727741843e3a64a7690e39455d59433d1fe Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 07:41:56 -0700 Subject: [PATCH 02/15] Add newline to end of file --- sdk/trace/batch_span_processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 56946aa3dba..6dc3083cce8 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -286,4 +286,4 @@ func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) { err := bsp.ForceFlush(ctx) assert.Equal(t, context.Canceled, err) -} \ No newline at end of file +} From 54e1a6da436d800702a7544548a45e2dd5690f6d Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 08:00:52 -0700 Subject: [PATCH 03/15] Change PR number in changelog for now --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c63578d0dd..709b0fc711d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Jaeger exporter was updated to use thrift v0.14.1. (#1712) - Migrate from using internally built and maintained version of the OTLP to the one hosted at `go.opentelemetry.io/proto/otlp`. (#1713) - Migrate from using `github.com/gogo/protobuf` to `google.golang.org/protobuf` to match `go.opentelemetry.io/proto/otlp`. (#1713) -- Modify `BatchSpanProcessor.ForceFlush` to abort after timeout/cancellation (#1618) +- Modify `BatchSpanProcessor.ForceFlush` to abort after timeout/cancellation (#TBD) ### Removed From 65cf9343e15cc67721ee154648510b3328287a89 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 08:42:46 -0700 Subject: [PATCH 04/15] Fix up comment --- sdk/trace/batch_span_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/trace/batch_span_processor.go b/sdk/trace/batch_span_processor.go index 616a4c9eb69..25d556e5f35 100644 --- a/sdk/trace/batch_span_processor.go +++ b/sdk/trace/batch_span_processor.go @@ -157,7 +157,7 @@ func (bsp *batchSpanProcessor) ForceFlush(ctx context.Context) error { } close(wait) }() - // Wait until the wait group is done or the context is cancelled/timed out + // Wait until the export is finished or the context is cancelled/timed out select { case <-wait: case <-ctx.Done(): From aedba6630a86c35c03ac25cec00babd549cb407c Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 14:34:07 -0700 Subject: [PATCH 05/15] Extend unit test to show that ForceFlush works and succeeds --- CHANGELOG.md | 2 +- sdk/trace/batch_span_processor_test.go | 34 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5bd87446b4..c562cbecbec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm The existing `ParentSpanID` and `HasRemoteParent` fields are removed in favor of this. (#1748) - The `ParentContext` field of the `"go.opentelemetry.io/otel/sdk/trace".SamplingParameters` is updated to hold a `context.Context` containing the parent span. This changes it to make `SamplingParameters` conform with the OpenTelemetry specification. (#1749) -- Modify `BatchSpanProcessor.ForceFlush` to abort after timeout/cancellation (#1757) +- Modify `BatchSpanProcessor.ForceFlush` to abort after timeout/cancellation. (#1757) ### Removed diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 6dc3083cce8..cfc1d8832ee 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -259,11 +259,37 @@ func TestBatchSpanProcessorShutdown(t *testing.T) { } func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { - var bp testBatchExporter - bsp := sdktrace.NewBatchSpanProcessor(&bp) + te := testBatchExporter{} + tp := basicTracerProvider(t) + option := testOption{ + name: "default BatchSpanProcessorOptions", + wantNumSpans: 2053, + wantBatchCount: 4, + genNumSpans: 2053, + } + ssp := createAndRegisterBatchSP(option, &te) + if ssp == nil { + t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) + } + tp.RegisterSpanProcessor(ssp) + tr := tp.Tracer("BatchSpanProcessorWithOption") + generateSpan(t, option.parallel, tr, option) + + // Force flush any held span batches + err := ssp.ForceFlush(context.Background()) - err := bsp.ForceFlush(context.Background()) - assert.Equal(t, nil, err) + gotNumOfSpans := te.len() + if option.wantNumSpans != gotNumOfSpans { + t.Errorf("number of exported span: got %+v, want %+v\n", + gotNumOfSpans, option.wantNumSpans) + } + gotBatchCount := te.getBatchCount() + if gotBatchCount < option.wantBatchCount { + t.Errorf("number batches: got %+v, want >= %+v\n", + gotBatchCount, option.wantBatchCount) + t.Errorf("Batches %v\n", te.sizes) + } + assert.NoError(t, err) } func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { From c0660370afebe8e9dd8e617bbcaacfde29ff623c Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 15:29:48 -0700 Subject: [PATCH 06/15] Add more options to make test more consistent --- sdk/trace/batch_span_processor_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index cfc1d8832ee..22ad9aa3dd0 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -263,6 +263,9 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { tp := basicTracerProvider(t) option := testOption{ name: "default BatchSpanProcessorOptions", + o: []sdktrace.BatchSpanProcessorOption{ + sdktrace.WithMaxQueueSize(0), + }, wantNumSpans: 2053, wantBatchCount: 4, genNumSpans: 2053, From d60340930aa383e326f16674f823366d3515a0a1 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 15:37:53 -0700 Subject: [PATCH 07/15] Add space for readability --- sdk/trace/batch_span_processor_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 22ad9aa3dd0..9e949a96e00 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -270,6 +270,7 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { wantBatchCount: 4, genNumSpans: 2053, } + ssp := createAndRegisterBatchSP(option, &te) if ssp == nil { t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) From 38860198f05b736a11da037b2d6534a63bce1b26 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 15:55:30 -0700 Subject: [PATCH 08/15] Lower spans to make test clearer --- sdk/trace/batch_span_processor_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 9e949a96e00..6619cfa7dcd 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -265,12 +265,12 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { name: "default BatchSpanProcessorOptions", o: []sdktrace.BatchSpanProcessorOption{ sdktrace.WithMaxQueueSize(0), + sdktrace.WithMaxExportBatchSize(3000), }, - wantNumSpans: 2053, - wantBatchCount: 4, - genNumSpans: 2053, + wantNumSpans: 512, + wantBatchCount: 1, + genNumSpans: 512, } - ssp := createAndRegisterBatchSP(option, &te) if ssp == nil { t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) From 6ed25e6e2b7f667007e5ae0dd1fdb4877f1c4dae Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 17:25:11 -0700 Subject: [PATCH 09/15] Change number of spans in test --- sdk/trace/batch_span_processor_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 6619cfa7dcd..405bb38b795 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -265,11 +265,10 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { name: "default BatchSpanProcessorOptions", o: []sdktrace.BatchSpanProcessorOption{ sdktrace.WithMaxQueueSize(0), - sdktrace.WithMaxExportBatchSize(3000), }, - wantNumSpans: 512, + wantNumSpans: 517, wantBatchCount: 1, - genNumSpans: 512, + genNumSpans: 517, } ssp := createAndRegisterBatchSP(option, &te) if ssp == nil { From 2403c5827473faba7e68c14842b573b9989deb1f Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 30 Mar 2021 22:51:39 -0700 Subject: [PATCH 10/15] Include a goroutine for the forceflush --- sdk/trace/batch_span_processor_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 405bb38b795..31a0804c694 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -278,8 +278,14 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { tr := tp.Tracer("BatchSpanProcessorWithOption") generateSpan(t, option.parallel, tr, option) - // Force flush any held span batches - err := ssp.ForceFlush(context.Background()) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // Force flush any held span batches + ssp.ForceFlush(context.Background()) + }() + wg.Wait() gotNumOfSpans := te.len() if option.wantNumSpans != gotNumOfSpans { @@ -292,7 +298,6 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { gotBatchCount, option.wantBatchCount) t.Errorf("Batches %v\n", te.sizes) } - assert.NoError(t, err) } func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { From 5a9d6030b373483a516fa4822017e45ae7e3d3b6 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 31 Mar 2021 08:57:20 -0700 Subject: [PATCH 11/15] Ran gofmt on test file --- sdk/trace/batch_span_processor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 31a0804c694..65421a7206c 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -262,7 +262,7 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { te := testBatchExporter{} tp := basicTracerProvider(t) option := testOption{ - name: "default BatchSpanProcessorOptions", + name: "default BatchSpanProcessorOptions", o: []sdktrace.BatchSpanProcessorOption{ sdktrace.WithMaxQueueSize(0), }, From dac4b35bbd258db7233eb2bce3885037ee6e38b7 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 31 Mar 2021 09:55:29 -0700 Subject: [PATCH 12/15] Try waiting for generating span to finish --- sdk/trace/batch_span_processor_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 65421a7206c..47a3229a44d 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -265,10 +265,11 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { name: "default BatchSpanProcessorOptions", o: []sdktrace.BatchSpanProcessorOption{ sdktrace.WithMaxQueueSize(0), + sdktrace.WithMaxExportBatchSize(10), }, - wantNumSpans: 517, + wantNumSpans: 6, wantBatchCount: 1, - genNumSpans: 517, + genNumSpans: 6, } ssp := createAndRegisterBatchSP(option, &te) if ssp == nil { @@ -276,17 +277,18 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { } tp.RegisterSpanProcessor(ssp) tr := tp.Tracer("BatchSpanProcessorWithOption") - generateSpan(t, option.parallel, tr, option) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() - // Force flush any held span batches - ssp.ForceFlush(context.Background()) + generateSpan(t, option.parallel, tr, option) }() wg.Wait() + // Force flush any held span batches + err := ssp.ForceFlush(context.Background()) + gotNumOfSpans := te.len() if option.wantNumSpans != gotNumOfSpans { t.Errorf("number of exported span: got %+v, want %+v\n", @@ -298,6 +300,7 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { gotBatchCount, option.wantBatchCount) t.Errorf("Batches %v\n", te.sizes) } + assert.NoError(t, err) } func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { From 7e4bc4026c0061da556ac96afd1d3a3f63c591bc Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 31 Mar 2021 10:13:24 -0700 Subject: [PATCH 13/15] Test to see if number of exported spans is within a range --- sdk/trace/batch_span_processor_test.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 47a3229a44d..9b1c990b13a 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -17,6 +17,7 @@ package trace_test import ( "context" "encoding/binary" + "math" "sync" "testing" "time" @@ -265,11 +266,11 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { name: "default BatchSpanProcessorOptions", o: []sdktrace.BatchSpanProcessorOption{ sdktrace.WithMaxQueueSize(0), - sdktrace.WithMaxExportBatchSize(10), + sdktrace.WithMaxExportBatchSize(3000), }, - wantNumSpans: 6, + wantNumSpans: 2053, wantBatchCount: 1, - genNumSpans: 6, + genNumSpans: 2053, } ssp := createAndRegisterBatchSP(option, &te) if ssp == nil { @@ -277,20 +278,13 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { } tp.RegisterSpanProcessor(ssp) tr := tp.Tracer("BatchSpanProcessorWithOption") - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - generateSpan(t, option.parallel, tr, option) - }() - wg.Wait() + generateSpan(t, option.parallel, tr, option) // Force flush any held span batches err := ssp.ForceFlush(context.Background()) gotNumOfSpans := te.len() - if option.wantNumSpans != gotNumOfSpans { + if math.Abs(float64(option.wantNumSpans-gotNumOfSpans)) > 10 { t.Errorf("number of exported span: got %+v, want %+v\n", gotNumOfSpans, option.wantNumSpans) } From 1b4271e4d5b6991a22346905a04d2b949c06b457 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 31 Mar 2021 10:30:02 -0700 Subject: [PATCH 14/15] Improve test feedback to give more detail about what went wrong --- sdk/trace/batch_span_processor_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 9b1c990b13a..78c643d69d8 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -17,6 +17,7 @@ package trace_test import ( "context" "encoding/binary" + "errors" "math" "sync" "testing" @@ -301,11 +302,15 @@ func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { var bp testBatchExporter bsp := sdktrace.NewBatchSpanProcessor(&bp) // Add timeout to context to test deadline - ctx, cancel := context.WithTimeout(context.Background(), 0) + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) defer cancel() + <-ctx.Done() - err := bsp.ForceFlush(ctx) - assert.Equal(t, context.DeadlineExceeded, err) + if err := bsp.ForceFlush(ctx); err == nil { + t.Error("expected context DeadlineExceeded error, got nil") + } else if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected context DeadlineExceeded error, got %v", err) + } } func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) { @@ -315,6 +320,9 @@ func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) { // Cancel the context cancel() - err := bsp.ForceFlush(ctx) - assert.Equal(t, context.Canceled, err) + if err := bsp.ForceFlush(ctx); err == nil { + t.Error("expected context canceled error, got nil") + } else if !errors.Is(err, context.Canceled) { + t.Errorf("expected context canceled error, got %v", err) + } } From 776fb174bb2bcca0154a7a40faaf00683092dd1b Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 31 Mar 2021 10:44:44 -0700 Subject: [PATCH 15/15] Take out absolute value difference and test only if less than and within 10 --- sdk/trace/batch_span_processor_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/trace/batch_span_processor_test.go b/sdk/trace/batch_span_processor_test.go index 78c643d69d8..f2bf428e22a 100644 --- a/sdk/trace/batch_span_processor_test.go +++ b/sdk/trace/batch_span_processor_test.go @@ -18,7 +18,6 @@ import ( "context" "encoding/binary" "errors" - "math" "sync" "testing" "time" @@ -285,8 +284,9 @@ func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { err := ssp.ForceFlush(context.Background()) gotNumOfSpans := te.len() - if math.Abs(float64(option.wantNumSpans-gotNumOfSpans)) > 10 { - t.Errorf("number of exported span: got %+v, want %+v\n", + spanDifference := option.wantNumSpans - gotNumOfSpans + if spanDifference > 10 || spanDifference < 0 { + t.Errorf("number of exported span not equal to or within 10 less than: got %+v, want %+v\n", gotNumOfSpans, option.wantNumSpans) } gotBatchCount := te.getBatchCount()