From 0f2c7e09ec31034f238924eac3e071e79cb1ebab Mon Sep 17 00:00:00 2001 From: Smuuf Date: Mon, 2 Mar 2020 23:34:28 +0100 Subject: [PATCH] New benchmark system. Two files that do rouglhly the same thing, one written in Primi and one written in PHP. Primi performance is now evaluated "how many times is Primi slower than an equivalent code in PHP?" This should make benchmarks more relevant - and any performance improvements or regressions should be visible better. Currently on my machine: 02.03.2020 22:35, 27.73, perf 37.86x slower vs 0.47678733 ... this is inclusing changes that are not yet commited. --- bin/bench | 69 +++++++++++---- example/bench_all.primi | 158 ---------------------------------- example/bench_simple.primi | 13 --- tests/bench/bench_perf.primi | 38 ++++++++ tests/bench/standard_perf.php | 68 +++++++++++++++ 5 files changed, 156 insertions(+), 190 deletions(-) delete mode 100644 example/bench_all.primi delete mode 100644 example/bench_simple.primi create mode 100644 tests/bench/bench_perf.primi create mode 100644 tests/bench/standard_perf.php diff --git a/bin/bench b/bin/bench index 43785b41..d3215e13 100755 --- a/bin/bench +++ b/bin/bench @@ -5,38 +5,69 @@ cd $(dirname $0)/.. # Time format for 'time' command. TIMEFORMAT=%R -iterations=10 -echo "Iterations: $iterations" - echo "Clearing AST cache ..." rm ./temp/ast*.json 2>/dev/null rm bench.out 2>/dev/null -simplefiles=$(find ./example -iname "bench*.primi") -simpletime=0 +ITERS=3 +PERF_STANDARD_PATH='./tests/bench/perf_standard.php' +BENCH_FILES=$(find ./tests/bench/ -iname "bench*.primi") +SIMPLE_TIME=0 +TOTAL_TIME=0 +AVG_SCORE=0 + +function get_precise_time { + date +%s.%N +} + +function measure_time { + # We probably should measure only user+kernel time our process really took + # (without I/O waits and stuff), but /usr/bin/time, which is handy, returns + # insufficient precision (only 2 decimal points). + # So lets measure everything. + START=`get_precise_time` + TMP=`$1 > /dev/null` + END=`get_precise_time` + perl -e "printf('%.8f', $END - $START);" +} + +function timeit_php { + echo `measure_time "php $1"` +} -function timeit { - echo `{ time ./primi $1 >> bench.out; } 2>&1` +function timeit_primi { + echo `measure_time "./primi $1"` } -echo "Running benchmark ..." -for i in $(seq $iterations) +echo -n "Measuring perf standard ... " +PERF_STD=`timeit_php $PERF_STANDARD_PATH` +echo "$PERF_STD s" + +echo "Running benchmarks ..." +for i in $(seq $ITERS) do - echo "█ ($i / $iterations) " - for f in $simplefiles + [[ "$i" == "1" ]] && STATE='parsing' || STATE='cached' + + [[ "$STATE" == "parsing" ]] && DESC="With AST parsing" || DESC="With cached AST" + echo "█ $DESC ($i / $ITERS)" + + for f in $BENCH_FILES do echo -n "$f ... " - tmp=$(timeit $f) - echo "$tmp s"; - simpletime=$(perl -e "printf('%.2f', $simpletime + $tmp);") + SIMPLE_TIME=$(timeit_primi $f) + SCORE=$(perl -e "printf('%.2f', $SIMPLE_TIME / $PERF_STD)") + TOTAL_SCORE=$(perl -e "printf('%.2f', $SCORE + $TOTAL_TIME)") + TOTAL_TIME=$(perl -e "printf('%.2f', $SIMPLE_TIME + $TOTAL_TIME)") + echo "$SIMPLE_TIME s (${SCORE}x slower)"; done done +AVG_TIME=`perl -e "printf('%.2f', $TOTAL_TIME / $ITERS);"` +AVG_SCORE=`perl -e "printf('%.2f', $TOTAL_SCORE / $ITERS);"` printf \ "Results:\n"\ -"- Total: $simpletime"" s \n"\ -"- AVG : "`perl -e "printf('%.2f', $simpletime / $iterations);"`" s \n" +"- Total: $TOTAL_TIME s\n"\ +"- AVG : $AVG_TIME s\n" -nowdate=`date +"%d.%m.%Y %H:%M"` -average=`perl -e "printf('%.2f', $simpletime / $iterations);"` -echo "$nowdate,$iterations,$simpletime,$average" >> "bench_progress.csv" +TODAY=`date +"%d.%m.%Y %H:%M"` +echo "$TODAY, ${AVG_TIME//,/.}, perf ${AVG_SCORE//,/.}x slower vs ${PERF_STD//,/.}" >> "bench_progress.csv" diff --git a/example/bench_all.primi b/example/bench_all.primi deleted file mode 100644 index e598c06b..00000000 --- a/example/bench_all.primi +++ /dev/null @@ -1,158 +0,0 @@ - -a = 1 + 2; -b = to_number("1") + 2; -c = to_string(1) + "2"; -d = "x".to_number() + 1; -e = to_string(1) + "x"; -f = "x" + "1"; -g = "1" + "x"; -h = "abc" + " def"; -i = "ahojahojahoj" - "ahoj"; -j = "1 2 3 haha 4 5 6" - "a"; -k = "tohle, s písmenem eř, je nejlepší věta evéř" - "ř"; -a = [1, 2, 3]; -b = [4,5,6]; -c = [a,b]; -d = [11:c, 22:a, 33:b]; -e = ["123456789123456789aaabbbcccdddeeefff": 987654321]; -f = [1,2,3,4,5].contains(3); -g = [1,2,3,4,5].contains(6); -h = [1,2,3,4,5].contains("ahoj"); -i = [1, 2, 3, 0: "ahoj",5].contains("ahoj"); -j = [1, 2, 3, 0: "ahoj",5].contains("neasi"); -k = [a[1], "foo", "somekey":"bar", "train"]; -l = k[0]; -m = k["somekey"]; -test = []; -nested = ["x": "y"]; -test.array_push(123); // test" should now contain 123 as the only item. -test_result_1 = test; // test_result_1" should now be the SAME value object as test" -test.array_push(456); -test_result_2 = test; -test.array_push(nested); -nested_2 = test.array_pop(); -nested_2.array_push("extra"); -test_result_3 = test; -test.array_pop(); -test.array_pop(); -n = ["a": "b", "c": "d"]["c"]; -o = ["x": "y", "c": "z"]["čaukoc" - "čauko"]; -p = [1, 2, 3]; -p[1] = 42; -a = [1, 2, 3]; -a[2] = 4; -b = ["a", "b", "c"]; -b[0] = ["x": "y"]; -c = ["i", "j", "k"]; -c[1] = [1: "a", 42: "b"]; -c[1][42] = "yeah"; -d = []; -d[] = "first"; -d[] = "second"; -d[] = []; -d[2][] = "nested"; -e = "ahoj"; -e[0] = "x"; -e[] = "ky"; -e[4] = "X"; -e_test1 = e[0]; -e_test2 = e[1]; -e_test3 = e[2]; -f = "something very supiš"; -f[] = "ko"; -a = 0; -function one() { - return 1; -} -function with_argument(arg) { - return arg / 4; -} -function no_return(a, b, c) { - a + 1; // Does not modify primary context. - b + 1; - c + 1; -} -no_return(a, 2, 3); -x = with_argument(one() * 16); -y = 0; -for (x in [1, 2, 3, 4, 5.6]) { - y = y + x; -} -n = ""; -length = 0; -for (m in "some long string with numbers 123") { - length = length + 1; - n = n + m; -} -a = 1 * 2; -b = "1" * 2; -c = "1.4" * 2; -d = 1 * "2"; -e = 1 * "2.4"; -f = 8 / 4; -g = 4 / 16; -h = 123 / to_number(a); -i = 123 / to_number(b); -j = (1 + (3 / (4 - 5)) + 2 / (37 * 2 / 8 - 42)); -k = 1 + 2 * 3 / 4 - 5 / 6 * 7 + 8; -a = "abcdefg"; -b = "abc123defg"; -c = "xyz456čaukomňauko"; -d = a == r"[cde]d"; -e = r"[cde]d" == a; -i = r"[c-e]"; -i_2 = r"[ce]"; -j = r"[0-9]+"; -k = r"[čau](ko)+mňau"; -xa = a - i; -xa_2 = a - i_2; -xb = b - i; -xb_2 = b - i_2; -xc = c - j; -xc_k = c - k; -l = "xoxoxomilanobergamo,anno:domini. Anno, neštvi ma.".string_replace(r"ann?o", "FAIL"); -m = "\\ahoj\n\\vo\\le" - r"\\ahoj\s"; -n = "a/b/c" - r"\/b"; // Test proper handling of escaped regex delimiters. -a = 4; -b = 5; -c = 6; -d = 7; -e = 4.25; -f = 5.25; -g = 6.25; -h = 7.25; -a = "ahojkyacauky"; -b = a.string_replace("a", "xxx"); -c = "voleneasijakprase".string_replace("ne", "jo"); -d = c.string_replace("asi", "určitě"); -e = "abcdefghijklmn".string_replace(r"[c-g]", "_"); -_definitions = [ - "á": "a", - "č": "c", - "ě": "e" -]; -f = "rádoby čau".string_replace(_definitions); -g = "vole".string_replace("v", "l").string_replace("o", "i") + "k"; -h = ("číslo je " + 000.cos().to_string()).string_replace(["č": "c", "í": "i"]); -i = "kokot je " + "oo".string_replace("o", "e"); -a_1 = 9.sqrt(); -b_1 = 3.pow(); -b_2 = 3.pow(2); -b_3 = 3.pow(4); -c = 0.cos(); -d = 0.sin(); -e = 1.3.ceil(); -f = 1.8.floor(); -g_1 = 1.6.round(); -g_2 = 1.5.round(); -g_3 = 1.49.round(); -a = 1; -b = 1234567890; -c = 0001234567890; -d = 0; -e = "2"; -f = "x"; -g = "xyz"; -i = "this is some whole very long sentence."; -j = true; -k = false; diff --git a/example/bench_simple.primi b/example/bench_simple.primi deleted file mode 100644 index 76722ce3..00000000 --- a/example/bench_simple.primi +++ /dev/null @@ -1,13 +0,0 @@ -// Average Execution Time (Xdebug OFF, cached AST) -// 2018/01/19 ... 0.34804 s -// 2018/03/15 ... 0.365 s -// 2018/04/16 ... 0.4821 s -fun = (x) => { return x + 1; } - -c = 0; -result = -10000; -while (c <= 10000) { - result = result + fun(c); - c = c + 1; -} - diff --git a/tests/bench/bench_perf.primi b/tests/bench/bench_perf.primi new file mode 100644 index 00000000..c4c284ab --- /dev/null +++ b/tests/bench/bench_perf.primi @@ -0,0 +1,38 @@ +max_iter = 500000 + +function bench_function_calls() { + + adder = function(x, y) { + return x + y + } + + result = -1024 + c = 0 + while (c < max_iter) { + result = result + adder(c, 1) + c = c + 1 + } + + return c + +} + +function bench_regex_matches() { + + haystack = "Když začínáme myslet, nemáme k dispozici nic jiného než myšlenku v " + + "její čisté neurčenosti, neboť k určení již patří jedno nebo nějaké " + + "jiné, ale na začátku ještě nemáme žádné jiné..." + + result = 0 + c = 0 + while (c < max_iter) { + result = result + (haystack == r"^.*(zač).*(,)?.*?(\.)").to_number() + c = c + 1 + } + + return c + +} + +bench_function_calls() +bench_regex_matches() diff --git a/tests/bench/standard_perf.php b/tests/bench/standard_perf.php new file mode 100644 index 00000000..8c86efbb --- /dev/null +++ b/tests/bench/standard_perf.php @@ -0,0 +1,68 @@ + measure('bench_function_calls'), + 'bench_regex_matches' => measure('bench_regex_matches'), +]; + +// +// Print results in INI format for easy parsing, if needed. +// + +$total = 0; +foreach ($results as $name => $time) { + echo "$name=$time\n"; + $total += $time; +} +echo "total=$total\n";