diff --git a/readme.md b/readme.md index ba2a3ec..e67342f 100644 --- a/readme.md +++ b/readme.md @@ -204,6 +204,27 @@ If the goal is to benchmark the initial calculation, it makes sense to place the If the goal, however, is to benchmark the entire process (initial calculation and subsequent caching), then it makes more sense to instantiate the object in classSetUp() so that it is only built once. +### Restricting Runtime + +If your benchmarks are running on different hardware or if your are using a generic benchmark +to test implementations with really different performance characteristics, then you might want +to restrict the runtime to ensure that you do not have to wait too long for the results. + +In this case you can use the `@maxRuntime` annotation with a number of seconds to restrict the runtime of a benchmark method. + +In the following example, the execution of the `slowIndexingAlgo()` stops once the desired number of iterations or a runtime of 5 minutes is reached: + +```php + /** + * @iterations 10000 + * @maxRuntime 300 + */ + public function slowIndexingAlgo() + { + $this->slow->index($this->data); + } +``` + ### Calibration Athletic uses Reflection and variable functions to invoke the methods in your Event. Because there is some internal overhead to variable functions, Athletic performs a "calibration" step before each iteration. This step calls an empty calibration method and times how long it takes. This time is then subtracted from the iterations total time, providing a more accurate total time. diff --git a/src/Athletic/AthleticEvent.php b/src/Athletic/AthleticEvent.php index 1bea574..8f1a372 100644 --- a/src/Athletic/AthleticEvent.php +++ b/src/Athletic/AthleticEvent.php @@ -18,6 +18,14 @@ */ abstract class AthleticEvent { + + /** + * The maximal number of iterations that is used for calibration. + * + * @var integer + */ + const MAX_CALIBRATION_ITERATIONS = 1000; + /** @var MethodResultsFactory */ private $methodResultsFactory; @@ -92,7 +100,7 @@ private function runBenchmarks($methods) $results = array(); foreach ($methods as $methodName => $annotations) { - if (isset($annotations['iterations']) === true) { + if (isset($annotations['iterations']) || isset($annotations['maxRuntime'])) { $results[] = $this->runMethodBenchmark($methodName, $annotations); } } @@ -102,23 +110,28 @@ private function runBenchmarks($methods) /** * @param string $method - * @param int $annotations + * @param Annotations $annotations * * @return MethodResults */ private function runMethodBenchmark($method, $annotations) { - $iterations = $annotations['iterations']; - $avgCalibration = $this->getCalibrationTime($iterations); + $iterations = isset($annotations['iterations']) ? $annotations['iterations'] : PHP_INT_MAX; + $maxRuntime = isset($annotations['maxRuntime']) ? $annotations['maxRuntime'] : PHP_INT_MAX; + $avgCalibration = $this->getCalibrationTime(min($iterations, static::MAX_CALIBRATION_ITERATIONS)); + $start = microtime(true); $results = array(); for ($i = 0; $i < $iterations; ++$i) { $this->setUp(); $results[$i] = $this->timeMethod($method) - $avgCalibration; $this->tearDown(); + if ((microtime(true) - $start) >= $maxRuntime) { + break; + } } - $finalResults = $this->methodResultsFactory->create($method, $results, $iterations); + $finalResults = $this->methodResultsFactory->create($method, $results, count($results)); $this->setOptionalAnnotations($finalResults, $annotations); diff --git a/tests/Athletic/AthleticEventTest.php b/tests/Athletic/AthleticEventTest.php index 2c52fcb..92b62e6 100644 --- a/tests/Athletic/AthleticEventTest.php +++ b/tests/Athletic/AthleticEventTest.php @@ -8,6 +8,7 @@ namespace Athletic; use Athletic\Results\MethodResults; +use Athletic\TestAsset\MaxRuntimeEvent; use Athletic\TestAsset\RunsCounter; use PHPUnit_Framework_TestCase; @@ -62,4 +63,50 @@ public function testCorrectRunsCount() $this->assertSame(5, $event->setUps); $this->assertSame(5, $event->tearDowns); } + + /** + * Ensures that a benchmark is aborted if the annotated maximal runtime is reached. + */ + public function testBenchmarkIsAbortedIfMaxRuntimeIsReached() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $event->run(); + + $this->assertEquals(1, $event->iterationAndRuntimeRuns); + } + + /** + * Ensures that the real number of iterations is shown in the result of a benchmark with + * runtime restriction (this does not have to be equal to the annotated number of iterations). + */ + public function testResultOfBenchmarkWithRuntimeRestrictionContainsCorrectNumberOfIterations() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $results = $event->run(); + + $this->assertInternalType('array', $results); + /* @var \Athletic\Results\MethodResults */ + $result = current($results); + $this->assertInstanceOf('Athletic\Results\MethodResults', $result); + $this->assertEquals(1, $result->iterations); + } + + /** + * Ensures that a benchmark is executed if it defines only a maximal runtime + * and no number of iterations. + */ + public function testBenchmarkIsExecutedIfOnlyRuntimeRestrictionIsSpecified() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $event->run(); + + $this->assertEquals(1, $event->onlyRuntimeRuns); + } + } diff --git a/tests/Athletic/TestAsset/MaxRuntimeEvent.php b/tests/Athletic/TestAsset/MaxRuntimeEvent.php new file mode 100644 index 0000000..63e6110 --- /dev/null +++ b/tests/Athletic/TestAsset/MaxRuntimeEvent.php @@ -0,0 +1,51 @@ + + * @package Athletic\TestAsset + */ +class MaxRuntimeEvent extends AthleticEvent +{ + + /** + * Counts how often testAdditionalRuntimeRestriction() was called. + * + * @var integer + */ + public $iterationAndRuntimeRuns = 0; + + /** + * Counts how often testOnlyRuntimeRestriction() was called. + * + * @var integer + */ + public $onlyRuntimeRuns = 0; + + /** + * Benchmark method with an additional runtime restriction in seconds. + * + * @Iterations 10 + * @MaxRuntime 0.0 + */ + public function testAdditionalRuntimeRestriction() + { + $this->iterationAndRuntimeRuns++; + } + + /** + * Benchmark method that defines only a runtime restriction. + * + * @MaxRuntime 0.0 + */ + public function testOnlyRuntimeRestriction() + { + $this->onlyRuntimeRuns++; + } + +}