From d3044f6520039b8ebdf34c537c5bdce78789b144 Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sat, 3 May 2014 19:00:20 +0200 Subject: [PATCH 1/7] fixed type hint --- src/Athletic/AthleticEvent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Athletic/AthleticEvent.php b/src/Athletic/AthleticEvent.php index 1bea574..d962bf7 100644 --- a/src/Athletic/AthleticEvent.php +++ b/src/Athletic/AthleticEvent.php @@ -102,7 +102,7 @@ private function runBenchmarks($methods) /** * @param string $method - * @param int $annotations + * @param Annotations $annotations * * @return MethodResults */ From b99c944dbd313241309240c130dbf05ecf300f98 Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sun, 4 May 2014 14:19:47 +0200 Subject: [PATCH 2/7] added tests that check runtime restriction --- tests/Athletic/AthleticEventTest.php | 29 +++++++++++ tests/Athletic/TestAsset/MaxRuntimeEvent.php | 51 ++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/Athletic/TestAsset/MaxRuntimeEvent.php diff --git a/tests/Athletic/AthleticEventTest.php b/tests/Athletic/AthleticEventTest.php index 2c52fcb..11ec6be 100644 --- a/tests/Athletic/AthleticEventTest.php +++ b/tests/Athletic/AthleticEventTest.php @@ -8,6 +8,7 @@ namespace Athletic; use Athletic\Results\MethodResults; +use Athletic\TestAsset\MaxRuntimeEvent; use Athletic\TestAsset\RunsCounter; use PHPUnit_Framework_TestCase; @@ -62,4 +63,32 @@ public function testCorrectRunsCount() $this->assertSame(5, $event->setUps); $this->assertSame(5, $event->tearDowns); } + + /** + * Ensures that a benchmark is aborted if the annotated maximal runtime is reached. + */ + public function testBenchmarkIsAbortedIfMaxRuntimeIsReached() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $event->run(); + + $this->assertEquals(1, $event->iterationAndRuntimeRuns); + } + + /** + * Ensures that a benchmark is executed if it defines only a maximal runtime + * and no number of iterations. + */ + public function testBenchmarkIsExecutedIfOnlyRuntimeRestrictionIsSpecified() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $event->run(); + + $this->assertEquals(1, $event->onlyRuntimeRuns); + } + } diff --git a/tests/Athletic/TestAsset/MaxRuntimeEvent.php b/tests/Athletic/TestAsset/MaxRuntimeEvent.php new file mode 100644 index 0000000..63e6110 --- /dev/null +++ b/tests/Athletic/TestAsset/MaxRuntimeEvent.php @@ -0,0 +1,51 @@ + + * @package Athletic\TestAsset + */ +class MaxRuntimeEvent extends AthleticEvent +{ + + /** + * Counts how often testAdditionalRuntimeRestriction() was called. + * + * @var integer + */ + public $iterationAndRuntimeRuns = 0; + + /** + * Counts how often testOnlyRuntimeRestriction() was called. + * + * @var integer + */ + public $onlyRuntimeRuns = 0; + + /** + * Benchmark method with an additional runtime restriction in seconds. + * + * @Iterations 10 + * @MaxRuntime 0.0 + */ + public function testAdditionalRuntimeRestriction() + { + $this->iterationAndRuntimeRuns++; + } + + /** + * Benchmark method that defines only a runtime restriction. + * + * @MaxRuntime 0.0 + */ + public function testOnlyRuntimeRestriction() + { + $this->onlyRuntimeRuns++; + } + +} From 53f07f506552f1a33e489078d0973d58c6f57a09 Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sun, 4 May 2014 14:26:38 +0200 Subject: [PATCH 3/7] added possibility to set a runtime restriction for a benchmark method --- src/Athletic/AthleticEvent.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Athletic/AthleticEvent.php b/src/Athletic/AthleticEvent.php index d962bf7..e5545da 100644 --- a/src/Athletic/AthleticEvent.php +++ b/src/Athletic/AthleticEvent.php @@ -92,7 +92,7 @@ private function runBenchmarks($methods) $results = array(); foreach ($methods as $methodName => $annotations) { - if (isset($annotations['iterations']) === true) { + if (isset($annotations['iterations']) || isset($annotations['maxRuntime'])) { $results[] = $this->runMethodBenchmark($methodName, $annotations); } } @@ -108,14 +108,19 @@ private function runBenchmarks($methods) */ private function runMethodBenchmark($method, $annotations) { - $iterations = $annotations['iterations']; - $avgCalibration = $this->getCalibrationTime($iterations); + $iterations = isset($annotations['iterations']) ? $annotations['iterations'] : PHP_INT_MAX; + $maxRuntime = isset($annotations['maxRuntime']) ? $annotations['maxRuntime'] : PHP_INT_MAX; + $avgCalibration = $this->getCalibrationTime(min($iterations, 1000)); + $start = microtime(true); $results = array(); for ($i = 0; $i < $iterations; ++$i) { $this->setUp(); $results[$i] = $this->timeMethod($method) - $avgCalibration; $this->tearDown(); + if ((microtime(true) - $start) >= $maxRuntime) { + break; + } } $finalResults = $this->methodResultsFactory->create($method, $results, $iterations); From ae65841309e710b85b2c399349c7264aa99e0c6f Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sun, 4 May 2014 17:37:45 +0200 Subject: [PATCH 4/7] added test that detects a problem with the number of iterations in the result (in combination with runtime restriction) --- tests/Athletic/AthleticEventTest.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/Athletic/AthleticEventTest.php b/tests/Athletic/AthleticEventTest.php index 11ec6be..92b62e6 100644 --- a/tests/Athletic/AthleticEventTest.php +++ b/tests/Athletic/AthleticEventTest.php @@ -77,6 +77,24 @@ public function testBenchmarkIsAbortedIfMaxRuntimeIsReached() $this->assertEquals(1, $event->iterationAndRuntimeRuns); } + /** + * Ensures that the real number of iterations is shown in the result of a benchmark with + * runtime restriction (this does not have to be equal to the annotated number of iterations). + */ + public function testResultOfBenchmarkWithRuntimeRestrictionContainsCorrectNumberOfIterations() + { + $event = new MaxRuntimeEvent(); + $event->setMethodFactory($this->resultsFactory); + + $results = $event->run(); + + $this->assertInternalType('array', $results); + /* @var \Athletic\Results\MethodResults */ + $result = current($results); + $this->assertInstanceOf('Athletic\Results\MethodResults', $result); + $this->assertEquals(1, $result->iterations); + } + /** * Ensures that a benchmark is executed if it defines only a maximal runtime * and no number of iterations. From 10dcb476ec51cae089e744206071cbeeaad0a6d7 Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sun, 4 May 2014 17:40:13 +0200 Subject: [PATCH 5/7] fixed number of iterations in result --- src/Athletic/AthleticEvent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Athletic/AthleticEvent.php b/src/Athletic/AthleticEvent.php index e5545da..4dd64d4 100644 --- a/src/Athletic/AthleticEvent.php +++ b/src/Athletic/AthleticEvent.php @@ -123,7 +123,7 @@ private function runMethodBenchmark($method, $annotations) } } - $finalResults = $this->methodResultsFactory->create($method, $results, $iterations); + $finalResults = $this->methodResultsFactory->create($method, $results, count($results)); $this->setOptionalAnnotations($finalResults, $annotations); From ee071cfd8532b9d2392185925bb5eb57c40d0eea Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sun, 25 May 2014 18:31:19 +0200 Subject: [PATCH 6/7] extracted constant --- src/Athletic/AthleticEvent.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Athletic/AthleticEvent.php b/src/Athletic/AthleticEvent.php index 4dd64d4..8f1a372 100644 --- a/src/Athletic/AthleticEvent.php +++ b/src/Athletic/AthleticEvent.php @@ -18,6 +18,14 @@ */ abstract class AthleticEvent { + + /** + * The maximal number of iterations that is used for calibration. + * + * @var integer + */ + const MAX_CALIBRATION_ITERATIONS = 1000; + /** @var MethodResultsFactory */ private $methodResultsFactory; @@ -110,7 +118,7 @@ private function runMethodBenchmark($method, $annotations) { $iterations = isset($annotations['iterations']) ? $annotations['iterations'] : PHP_INT_MAX; $maxRuntime = isset($annotations['maxRuntime']) ? $annotations['maxRuntime'] : PHP_INT_MAX; - $avgCalibration = $this->getCalibrationTime(min($iterations, 1000)); + $avgCalibration = $this->getCalibrationTime(min($iterations, static::MAX_CALIBRATION_ITERATIONS)); $start = microtime(true); $results = array(); From cbd99cad1eeaf5a3eb33f7fea8ccb59784d82f43 Mon Sep 17 00:00:00 2001 From: Matthias Molitor Date: Sat, 21 Jun 2014 14:42:37 +0200 Subject: [PATCH 7/7] documented @maxRuntime annotation --- readme.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/readme.md b/readme.md index ba2a3ec..e67342f 100644 --- a/readme.md +++ b/readme.md @@ -204,6 +204,27 @@ If the goal is to benchmark the initial calculation, it makes sense to place the If the goal, however, is to benchmark the entire process (initial calculation and subsequent caching), then it makes more sense to instantiate the object in classSetUp() so that it is only built once. +### Restricting Runtime + +If your benchmarks are running on different hardware or if your are using a generic benchmark +to test implementations with really different performance characteristics, then you might want +to restrict the runtime to ensure that you do not have to wait too long for the results. + +In this case you can use the `@maxRuntime` annotation with a number of seconds to restrict the runtime of a benchmark method. + +In the following example, the execution of the `slowIndexingAlgo()` stops once the desired number of iterations or a runtime of 5 minutes is reached: + +```php + /** + * @iterations 10000 + * @maxRuntime 300 + */ + public function slowIndexingAlgo() + { + $this->slow->index($this->data); + } +``` + ### Calibration Athletic uses Reflection and variable functions to invoke the methods in your Event. Because there is some internal overhead to variable functions, Athletic performs a "calibration" step before each iteration. This step calls an empty calibration method and times how long it takes. This time is then subtracted from the iterations total time, providing a more accurate total time.