Runtime Restriction of Benchmark Methods

readme.md

@@ -204,6 +204,27 @@ If the goal is to benchmark the initial calculation, it makes sense to place the
 
 If the goal, however, is to benchmark the entire process (initial calculation and subsequent caching), then it makes more sense to instantiate the object in classSetUp() so that it is only built once.
 
+### Restricting Runtime
+
+If your benchmarks are running on different hardware or if your are using a generic benchmark
+to test implementations with really different performance characteristics, then you might want
+to restrict the runtime to ensure that you do not have to wait too long for the results.
+
+In this case you can use the `@maxRuntime` annotation with a number of seconds to restrict  the runtime of a benchmark method.
+
+In the following example, the execution of the `slowIndexingAlgo()` stops once the desired number of iterations or a runtime of 5 minutes is reached:
+
+```php
+    /**
+     * @iterations 10000
+     * @maxRuntime 300
+     */
+    public function slowIndexingAlgo()
+    {
+        $this->slow->index($this->data);
+    }
+```
+
 ### Calibration
 
 Athletic uses Reflection and variable functions to invoke the methods in your Event.  Because there is some internal overhead to variable functions, Athletic performs a "calibration" step before each iteration.  This step calls an empty calibration method and times how long it takes.  This time is then subtracted from the iterations total time, providing a more accurate total time.

src/Athletic/AthleticEvent.php

@@ -18,6 +18,14 @@
  */
 abstract class AthleticEvent
 {
+
+    /**
+     * The maximal number of iterations that is used for calibration.
+     *
+     * @var integer
+     */
+    const MAX_CALIBRATION_ITERATIONS = 1000;
+
     /** @var  MethodResultsFactory */
     private $methodResultsFactory;
 
@@ -92,7 +100,7 @@ private function runBenchmarks($methods)
         $results = array();
 
         foreach ($methods as $methodName => $annotations) {
-            if (isset($annotations['iterations']) === true) {
+            if (isset($annotations['iterations']) || isset($annotations['maxRuntime'])) {
                 $results[] = $this->runMethodBenchmark($methodName, $annotations);
             }
         }
@@ -102,23 +110,28 @@ private function runBenchmarks($methods)
 
     /**
      * @param string $method
-     * @param int    $annotations
+     * @param Annotations $annotations
      *
      * @return MethodResults
      */
     private function runMethodBenchmark($method, $annotations)
     {
-        $iterations = $annotations['iterations'];
-        $avgCalibration = $this->getCalibrationTime($iterations);
+        $iterations = isset($annotations['iterations']) ? $annotations['iterations'] : PHP_INT_MAX;
+        $maxRuntime = isset($annotations['maxRuntime']) ? $annotations['maxRuntime'] : PHP_INT_MAX;
+        $avgCalibration = $this->getCalibrationTime(min($iterations, static::MAX_CALIBRATION_ITERATIONS));
 
+        $start = microtime(true);
         $results = array();
         for ($i = 0; $i < $iterations; ++$i) {
             $this->setUp();
             $results[$i] = $this->timeMethod($method) - $avgCalibration;
             $this->tearDown();
+            if ((microtime(true) - $start) >= $maxRuntime) {
+                break;
+            }
         }
 
-        $finalResults = $this->methodResultsFactory->create($method, $results, $iterations);
+        $finalResults = $this->methodResultsFactory->create($method, $results, count($results));
 
         $this->setOptionalAnnotations($finalResults, $annotations);
 

tests/Athletic/AthleticEventTest.php

@@ -8,6 +8,7 @@
 namespace Athletic;
 
 use Athletic\Results\MethodResults;
+use Athletic\TestAsset\MaxRuntimeEvent;
 use Athletic\TestAsset\RunsCounter;
 use PHPUnit_Framework_TestCase;
 
@@ -62,4 +63,50 @@ public function testCorrectRunsCount()
         $this->assertSame(5, $event->setUps);
         $this->assertSame(5, $event->tearDowns);
     }
+
+    /**
+     * Ensures that a benchmark is aborted if the annotated maximal runtime is reached.
+     */
+    public function testBenchmarkIsAbortedIfMaxRuntimeIsReached()
+    {
+        $event = new MaxRuntimeEvent();
+        $event->setMethodFactory($this->resultsFactory);
+
+        $event->run();
+
+        $this->assertEquals(1, $event->iterationAndRuntimeRuns);
+    }
+
+    /**
+     * Ensures that the real number of iterations is shown in the result of a benchmark with
+     * runtime restriction (this does not have to be equal to the annotated number of iterations).
+     */
+    public function testResultOfBenchmarkWithRuntimeRestrictionContainsCorrectNumberOfIterations()
+    {
+        $event = new MaxRuntimeEvent();
+        $event->setMethodFactory($this->resultsFactory);
+
+        $results = $event->run();
+
+        $this->assertInternalType('array', $results);
+        /* @var \Athletic\Results\MethodResults */
+        $result = current($results);
+        $this->assertInstanceOf('Athletic\Results\MethodResults', $result);
+        $this->assertEquals(1, $result->iterations);
+    }
+
+    /**
+     * Ensures that a benchmark is executed if it defines only a maximal runtime
+     * and no number of iterations.
+     */
+    public function testBenchmarkIsExecutedIfOnlyRuntimeRestrictionIsSpecified()
+    {
+        $event = new MaxRuntimeEvent();
+        $event->setMethodFactory($this->resultsFactory);
+
+        $event->run();
+
+        $this->assertEquals(1, $event->onlyRuntimeRuns);
+    }
+
 }

tests/Athletic/TestAsset/MaxRuntimeEvent.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Athletic\TestAsset;
+
+use Athletic\AthleticEvent;
+
+/**
+ * Event that is used to test the runtime restriction feature.
+ *
+ * @author Matthias Molitor <[email protected]>
+ * @package Athletic\TestAsset
+ */
+class MaxRuntimeEvent extends AthleticEvent
+{
+
+    /**
+     * Counts how often testAdditionalRuntimeRestriction() was called.
+     *
+     * @var integer
+     */
+    public $iterationAndRuntimeRuns = 0;
+
+    /**
+     * Counts how often testOnlyRuntimeRestriction() was called.
+     *
+     * @var integer
+     */
+    public $onlyRuntimeRuns = 0;
+
+    /**
+     * Benchmark method with an additional runtime restriction in seconds.
+     *
+     * @Iterations 10
+     * @MaxRuntime 0.0
+     */
+    public function testAdditionalRuntimeRestriction()
+    {
+        $this->iterationAndRuntimeRuns++;
+    }
+
+    /**
+     * Benchmark method that defines only a runtime restriction.
+     *
+     * @MaxRuntime 0.0
+     */
+    public function testOnlyRuntimeRestriction()
+    {
+        $this->onlyRuntimeRuns++;
+    }
+
+}