From 77a9bcc402115cd6ec5e9f6806ebd194343465fe Mon Sep 17 00:00:00 2001 From: Adam Luzsi Date: Wed, 1 May 2024 22:09:44 +0200 Subject: [PATCH] add support for explicitly definning Benchmark cases When using testcase.Spec as a testing suite, you can also utilise it as a benchmark suite. The Spec.Benchmark feature enables you to define your benchmark cases separately from your testing scenarios, if desired. --- Spec.go | 93 +++++++++++++++++++++++++++++++++++++------ Spec_test.go | 100 +++++++++++++++++++++++++++++++++++++++++++++++ examples_test.go | 12 ++++++ opts.go | 6 +++ 4 files changed, 198 insertions(+), 13 deletions(-) diff --git a/Spec.go b/Spec.go index 26a8f27..ad88c9f 100644 --- a/Spec.go +++ b/Spec.go @@ -85,22 +85,27 @@ type Spec struct { defs []func(*Spec) - immutable bool - vars *variables - parallel bool - sequential bool + immutable bool + vars *variables + parallel bool + sequential bool + flaky *assert.Retry + eventually *assert.Retry + group *struct{ name string } + description string + tags []string + tests []func() + + hasRan bool + isTest bool + isBenchmark bool + + skipTest bool skipBenchmark bool - flaky *assert.Retry - eventually *assert.Retry - group *struct{ name string } - description string - tags []string - tests []func() finished bool orderer orderer seed int64 - isTest bool sync bool isSuite bool @@ -192,10 +197,29 @@ func (spec *Spec) Test(desc string, test tBlock, opts ...SpecOption) { } spec.testingTB.Helper() s := spec.newSubSpec(desc, opts...) - s.isTest = true + s.isTest = !s.isBenchmark + s.hasRan = true s.run(test) } +const panicMessageForRunningBenchmarkAfterTest = `when .Benchmark is defined, they either must be specified before any .Test call in the top level, or should be done under a context ` + +// Benchmark creates a becnhmark in the given Spec context. +// +// Creating a Benchmark will signal the Spec that test and benchmark happens seperately, and a test should not double as a benchmark. +func (spec *Spec) Benchmark(desc string, test tBlock, opts ...SpecOption) { + if spec.isTestRunner() { + return + } + if spec.sync && spec.hasTestRan() { + panic(panicMessageForRunningBenchmarkAfterTest) + } + spec.skipTest = true // flag test for skipping + opts = append([]SpecOption{}, opts...) + opts = append(opts, benchmark()) + spec.Test(desc, test, opts...) +} + const warnEventOnImmutableFormat = `you can't use .%s after you already used when/and/then` // Parallel allows you to set list test case for the spec where this is being called, @@ -258,6 +282,11 @@ func (spec *Spec) Tag(tags ...string) { func (spec *Spec) isAllowedToRun() bool { spec.testingTB.Helper() + + if spec.isTest && !spec.isTestAllowedToRun() { + return false + } + currentTagSet := spec.getTagSet() settings := getCachedTagSettings() @@ -277,9 +306,20 @@ func (spec *Spec) isAllowedToRun() bool { allowed = true } } + // TODO: Exclude return allowed } +func (spec *Spec) isTestAllowedToRun() bool { + spec.testingTB.Helper() + for _, context := range spec.specsFromParent() { + if context.skipTest { + return false + } + } + return true +} + func (spec *Spec) isBenchAllowedToRun() bool { spec.testingTB.Helper() for _, context := range spec.specsFromParent() { @@ -357,6 +397,9 @@ func (spec *Spec) run(blk func(*T)) { name := spec.name() switch tb := spec.testingTB.(type) { case tRunner: + if spec.isBenchmark { + return + } spec.addTest(func() { if h, ok := tb.(helper); ok { h.Helper() @@ -397,6 +440,17 @@ func (spec *Spec) run(blk func(*T)) { } } +func (spec *Spec) isTestRunner() bool { + switch spec.testingTB.(type) { + case bRunner: + return false + case tRunner, TBRunner: + return true + default: + return true + } +} + func (spec *Spec) getTestSeed(tb testing.TB) int64 { h := fnv.New64a() _, _ = h.Write([]byte(tb.Name())) @@ -407,6 +461,7 @@ func (spec *Spec) getTestSeed(tb testing.TB) int64 { func (spec *Spec) runTB(tb testing.TB, blk func(*T)) { spec.testingTB.Helper() tb.Helper() + spec.hasRan = true if tb, ok := tb.(interface{ Parallel() }); ok && spec.isParallel() { tb.Parallel() } @@ -568,7 +623,7 @@ func (spec *Spec) specsFromCurrent() []*Spec { func (spec *Spec) lookupParent() (*Spec, bool) { spec.testingTB.Helper() for _, s := range spec.specsFromCurrent() { - if s.isTest { // skip test + if s.hasRan { // skip test continue } if s == spec { // skip self @@ -638,6 +693,18 @@ func (spec *Spec) getIsSuite() bool { return false } +func (spec *Spec) hasTestRan() bool { + if spec.isTest && spec.hasRan { + return true + } + for _, child := range spec.children { + if child.isTest && child.hasRan { + return true + } + } + return false +} + func checkSuite(tb testing.TB, opts []SpecOption) (testing.TB, []SpecOption) { if tb == nil { return internal.SuiteNullTB{}, append(opts, AsSuite()) diff --git a/Spec_test.go b/Spec_test.go index cdf8f30..bbd75a8 100644 --- a/Spec_test.go +++ b/Spec_test.go @@ -1544,3 +1544,103 @@ func TestRunSuite_spectAsSuite(t *testing.T) { assert.True(t, strings.HasSuffix(name1, "Suite-namE-1/tst1")) assert.True(t, strings.HasSuffix(name2, "Suite-namE-2/tst2")) } + +func TestSpec_Benchmark(t *testing.T) { + var testRan, benchRan bool + s := testcase.NewSpec(t) + s.Benchmark("", func(t *testcase.T) { + benchRan = true + t.Skip() + }) + s.Test("", func(t *testcase.T) { + testRan = true + t.Skip() + }) + s.Finish() + assert.True(t, testRan) + assert.False(t, benchRan) +} + +func BenchmarkTestSpec_Benchmark(b *testing.B) { + b.Run("sync - happy", func(b *testing.B) { + var testRan, benchRan1, benchRan2 bool + s := testcase.NewSpec(b) + s.Benchmark("", func(t *testcase.T) { + benchRan1 = true + t.Skip() + }) + s.Benchmark("", func(t *testcase.T) { + benchRan2 = true + t.Skip() + }) + s.Test("", func(t *testcase.T) { + testRan = true + t.Skip() + }) + s.Finish() + assert.False(b, testRan) + assert.True(b, benchRan1) + assert.True(b, benchRan2) + b.Skip("done") + }) + b.Run("hook", func(b *testing.B) { + var hookRan, benchRan bool + s := testcase.NewSpec(b) + s.Before(func(t *testcase.T) { + hookRan = true + }) + s.Benchmark("", func(t *testcase.T) { + benchRan = true + t.Skip() + }) + s.Finish() + assert.True(b, hookRan) + assert.True(b, benchRan) + b.Skip("done") + }) + b.Run("sync - incorrect order panics", func(b *testing.B) { + s := testcase.NewSpec(b) + s.Test("", func(t *testcase.T) { t.Skip() }) + assert.Panic(b, func() { + s.Benchmark("", func(t *testcase.T) { + t.Skip() + }) + }, "expect to panic when Benchmark used after Test in sync mode") + b.Skip("done") + }) + b.Run("sync - if top level benchmark is defined, sub context tests are not used as Benchmark", func(b *testing.B) { + var testRan1, testRan2, benchRan1, benchRan2 bool + s := testcase.NewSpec(b) + s.Describe("d1", func(s *testcase.Spec) { + s.Benchmark("b1", func(t *testcase.T) { + benchRan1 = true + t.Skip() + }) + s.Test("t1", func(t *testcase.T) { + testRan1 = true + t.Skip() + }) + s.When("x", func(s *testcase.Spec) { + s.Benchmark("b2", func(t *testcase.T) { + benchRan2 = true + t.Skip() + }) + }) + s.When("y", func(s *testcase.Spec) { + s.Test("t2", func(t *testcase.T) { + testRan2 = true + panic("boom") + }) + }) + s.Context("c", func(s *testcase.Spec) { + s.Test("t2", func(t *testcase.T) { panic("boom") }) + }) + }) + s.Finish() + assert.Equal(b, testRan1, false) + assert.Equal(b, testRan2, false) + assert.Equal(b, benchRan1, true) + assert.Equal(b, benchRan2, true) + b.Skip("done") + }) +} diff --git a/examples_test.go b/examples_test.go index 435126e..392dbd6 100644 --- a/examples_test.go +++ b/examples_test.go @@ -1543,3 +1543,15 @@ func exampleOpenSuite() testcase.OpenSuite { }) return s.AsSuite() } + +func ExampleSpec_Benchmark() { + s := testcase.NewSpec(nil) + + s.Before(func(t *testcase.T) { + // arrangement for everything, including the Benchmark + }) + + s.Benchmark("bench scenario", func(t *testcase.T) { + // OK + }) +} diff --git a/opts.go b/opts.go index 590c507..cbd3016 100644 --- a/opts.go +++ b/opts.go @@ -92,6 +92,12 @@ func sequential() SpecOption { }) } +func benchmark() SpecOption { + return specOptionFunc(func(s *Spec) { + s.isBenchmark = true + }) +} + type SpecOption interface { setup(*Spec) }