add support for explicitly definning Benchmark cases

When using testcase.Spec as a testing suite, you can also utilise it as a benchmark suite.
The Spec.Benchmark feature enables you to define your benchmark cases separately from your testing scenarios, if desired.

Spec.go

@@ -85,22 +85,27 @@ type Spec struct {
 
 	defs []func(*Spec)
 
-	immutable     bool
-	vars          *variables
-	parallel      bool
-	sequential    bool
+	immutable   bool
+	vars        *variables
+	parallel    bool
+	sequential  bool
+	flaky       *assert.Retry
+	eventually  *assert.Retry
+	group       *struct{ name string }
+	description string
+	tags        []string
+	tests       []func()
+
+	hasRan      bool
+	isTest      bool
+	isBenchmark bool
+
+	skipTest      bool
 	skipBenchmark bool
-	flaky         *assert.Retry
-	eventually    *assert.Retry
-	group         *struct{ name string }
-	description   string
-	tags          []string
-	tests         []func()
 
 	finished bool
 	orderer  orderer
 	seed     int64
-	isTest   bool
 	sync     bool
 
 	isSuite   bool
@@ -192,10 +197,29 @@ func (spec *Spec) Test(desc string, test tBlock, opts ...SpecOption) {
 	}
 	spec.testingTB.Helper()
 	s := spec.newSubSpec(desc, opts...)
-	s.isTest = true
+	s.isTest = !s.isBenchmark
+	s.hasRan = true
 	s.run(test)
 }
 
+const panicMessageForRunningBenchmarkAfterTest = `when .Benchmark is defined, they either must be specified before any .Test call in the top level, or should be done under a context `
+
+// Benchmark creates a becnhmark in the given Spec context.
+//
+// Creating a Benchmark will signal the Spec that test and benchmark happens seperately, and a test should not double as a benchmark.
+func (spec *Spec) Benchmark(desc string, test tBlock, opts ...SpecOption) {
+	if spec.isTestRunner() {
+		return
+	}
+	if spec.sync && spec.hasTestRan() {
+		panic(panicMessageForRunningBenchmarkAfterTest)
+	}
+	spec.skipTest = true // flag test for skipping
+	opts = append([]SpecOption{}, opts...)
+	opts = append(opts, benchmark())
+	spec.Test(desc, test, opts...)
+}
+
 const warnEventOnImmutableFormat = `you can't use .%s after you already used when/and/then`
 
 // Parallel allows you to set list test case for the spec where this is being called,
@@ -258,6 +282,11 @@ func (spec *Spec) Tag(tags ...string) {
 
 func (spec *Spec) isAllowedToRun() bool {
 	spec.testingTB.Helper()
+
+	if spec.isTest && !spec.isTestAllowedToRun() {
+		return false
+	}
+
 	currentTagSet := spec.getTagSet()
 	settings := getCachedTagSettings()
 
@@ -277,9 +306,20 @@ func (spec *Spec) isAllowedToRun() bool {
 			allowed = true
 		}
 	}
+	// TODO: Exclude
 	return allowed
 }
 
+func (spec *Spec) isTestAllowedToRun() bool {
+	spec.testingTB.Helper()
+	for _, context := range spec.specsFromParent() {
+		if context.skipTest {
+			return false
+		}
+	}
+	return true
+}
+
 func (spec *Spec) isBenchAllowedToRun() bool {
 	spec.testingTB.Helper()
 	for _, context := range spec.specsFromParent() {
@@ -357,6 +397,9 @@ func (spec *Spec) run(blk func(*T)) {
 	name := spec.name()
 	switch tb := spec.testingTB.(type) {
 	case tRunner:
+		if spec.isBenchmark {
+			return
+		}
 		spec.addTest(func() {
 			if h, ok := tb.(helper); ok {
 				h.Helper()
@@ -397,6 +440,17 @@ func (spec *Spec) run(blk func(*T)) {
 	}
 }
 
+func (spec *Spec) isTestRunner() bool {
+	switch spec.testingTB.(type) {
+	case bRunner:
+		return false
+	case tRunner, TBRunner:
+		return true
+	default:
+		return true
+	}
+}
+
 func (spec *Spec) getTestSeed(tb testing.TB) int64 {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(tb.Name()))
@@ -407,6 +461,7 @@ func (spec *Spec) getTestSeed(tb testing.TB) int64 {
 func (spec *Spec) runTB(tb testing.TB, blk func(*T)) {
 	spec.testingTB.Helper()
 	tb.Helper()
+	spec.hasRan = true
 	if tb, ok := tb.(interface{ Parallel() }); ok && spec.isParallel() {
 		tb.Parallel()
 	}
@@ -568,7 +623,7 @@ func (spec *Spec) specsFromCurrent() []*Spec {
 func (spec *Spec) lookupParent() (*Spec, bool) {
 	spec.testingTB.Helper()
 	for _, s := range spec.specsFromCurrent() {
-		if s.isTest { // skip test
+		if s.hasRan { // skip test
 			continue
 		}
 		if s == spec { // skip self
@@ -638,6 +693,18 @@ func (spec *Spec) getIsSuite() bool {
 	return false
 }
 
+func (spec *Spec) hasTestRan() bool {
+	if spec.isTest && spec.hasRan {
+		return true
+	}
+	for _, child := range spec.children {
+		if child.isTest && child.hasRan {
+			return true
+		}
+	}
+	return false
+}
+
 func checkSuite(tb testing.TB, opts []SpecOption) (testing.TB, []SpecOption) {
 	if tb == nil {
 		return internal.SuiteNullTB{}, append(opts, AsSuite())

Spec_test.go

@@ -1544,3 +1544,103 @@ func TestRunSuite_spectAsSuite(t *testing.T) {
 	assert.True(t, strings.HasSuffix(name1, "Suite-namE-1/tst1"))
 	assert.True(t, strings.HasSuffix(name2, "Suite-namE-2/tst2"))
 }
+
+func TestSpec_Benchmark(t *testing.T) {
+	var testRan, benchRan bool
+	s := testcase.NewSpec(t)
+	s.Benchmark("", func(t *testcase.T) {
+		benchRan = true
+		t.Skip()
+	})
+	s.Test("", func(t *testcase.T) {
+		testRan = true
+		t.Skip()
+	})
+	s.Finish()
+	assert.True(t, testRan)
+	assert.False(t, benchRan)
+}
+
+func BenchmarkTestSpec_Benchmark(b *testing.B) {
+	b.Run("sync - happy", func(b *testing.B) {
+		var testRan, benchRan1, benchRan2 bool
+		s := testcase.NewSpec(b)
+		s.Benchmark("", func(t *testcase.T) {
+			benchRan1 = true
+			t.Skip()
+		})
+		s.Benchmark("", func(t *testcase.T) {
+			benchRan2 = true
+			t.Skip()
+		})
+		s.Test("", func(t *testcase.T) {
+			testRan = true
+			t.Skip()
+		})
+		s.Finish()
+		assert.False(b, testRan)
+		assert.True(b, benchRan1)
+		assert.True(b, benchRan2)
+		b.Skip("done")
+	})
+	b.Run("hook", func(b *testing.B) {
+		var hookRan, benchRan bool
+		s := testcase.NewSpec(b)
+		s.Before(func(t *testcase.T) {
+			hookRan = true
+		})
+		s.Benchmark("", func(t *testcase.T) {
+			benchRan = true
+			t.Skip()
+		})
+		s.Finish()
+		assert.True(b, hookRan)
+		assert.True(b, benchRan)
+		b.Skip("done")
+	})
+	b.Run("sync - incorrect order panics", func(b *testing.B) {
+		s := testcase.NewSpec(b)
+		s.Test("", func(t *testcase.T) { t.Skip() })
+		assert.Panic(b, func() {
+			s.Benchmark("", func(t *testcase.T) {
+				t.Skip()
+			})
+		}, "expect to panic when Benchmark used after Test in sync mode")
+		b.Skip("done")
+	})
+	b.Run("sync - if top level benchmark is defined, sub context tests are not used as Benchmark", func(b *testing.B) {
+		var testRan1, testRan2, benchRan1, benchRan2 bool
+		s := testcase.NewSpec(b)
+		s.Describe("d1", func(s *testcase.Spec) {
+			s.Benchmark("b1", func(t *testcase.T) {
+				benchRan1 = true
+				t.Skip()
+			})
+			s.Test("t1", func(t *testcase.T) {
+				testRan1 = true
+				t.Skip()
+			})
+			s.When("x", func(s *testcase.Spec) {
+				s.Benchmark("b2", func(t *testcase.T) {
+					benchRan2 = true
+					t.Skip()
+				})
+			})
+			s.When("y", func(s *testcase.Spec) {
+				s.Test("t2", func(t *testcase.T) {
+					testRan2 = true
+					panic("boom")
+				})
+			})
+			s.Context("c", func(s *testcase.Spec) {
+				s.Test("t2", func(t *testcase.T) { panic("boom") })
+			})
+		})
+		s.Finish()
+		assert.Equal(b, testRan1, false)
+		assert.Equal(b, testRan2, false)
+		assert.Equal(b, benchRan1, true)
+		assert.Equal(b, benchRan2, true)
+		b.Skip("done")
+	})
+}

examples_test.go

@@ -1543,3 +1543,15 @@ func exampleOpenSuite() testcase.OpenSuite {
 	})
 	return s.AsSuite()
 }
+
+func ExampleSpec_Benchmark() {
+	s := testcase.NewSpec(nil)
+
+	s.Before(func(t *testcase.T) {
+		// arrangement for everything, including the Benchmark
+	})
+
+	s.Benchmark("bench scenario", func(t *testcase.T) {
+		// OK
+	})
+}

opts.go

@@ -92,6 +92,12 @@ func sequential() SpecOption {
 	})
 }
 
+func benchmark() SpecOption {
+	return specOptionFunc(func(s *Spec) {
+		s.isBenchmark = true
+	})
+}
+
 type SpecOption interface {
 	setup(*Spec)
 }