add test framework for benchmarks

.gitignore

@@ -12,6 +12,9 @@ compile_commands.json
 CTestTestfile.cmake
 build
 
+### Python
+__pycache__
+
 ### IDE
 # Clion
 .idea

benchmark/CMakeLists.txt

@@ -160,6 +160,9 @@ add_subdirectory(solver)
 add_subdirectory(sparse_blas)
 add_subdirectory(spmv)
 add_subdirectory(tools)
+if (GINKGO_BUILD_TESTS)
+    add_subdirectory(test)
+endif()
 
 configure_file(run_all_benchmarks.sh run_all_benchmarks.sh COPYONLY)
 

benchmark/blas/distributed/CMakeLists.txt

@@ -1 +1 @@
-ginkgo_add_typed_benchmark_executables(multi-vector-distributed "NO" multi_vector.cpp)
+ginkgo_add_typed_benchmark_executables(multi_vector_distributed "NO" multi_vector.cpp)

benchmark/conversions/CMakeLists.txt

@@ -1 +1 @@
-ginkgo_add_typed_benchmark_executables(conversions "NO" conversions.cpp)
+ginkgo_add_typed_benchmark_executables(conversion "NO" conversions.cpp)

benchmark/solver/distributed/CMakeLists.txt

@@ -1 +1 @@
-ginkgo_add_typed_benchmark_executables(solver-distributed "YES" solver.cpp)
+ginkgo_add_typed_benchmark_executables(solver_distributed "YES" solver.cpp)

benchmark/spmv/distributed/CMakeLists.txt

@@ -1 +1 @@
-ginkgo_add_typed_benchmark_executables(spmv-distributed "YES" spmv.cpp)
+ginkgo_add_typed_benchmark_executables(spmv_distributed "YES" spmv.cpp)

benchmark/test/CMakeLists.txt

@@ -0,0 +1,28 @@
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+function(add_benchmark_test test_name)
+    configure_file(${test_name}.py ${test_name}.py COPYONLY)
+    add_test(NAME benchmark_${test_name}
+             COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.py
+             WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    set(regenerate_target benchmark_test_${test_name}_regenerate)
+    add_custom_target(${regenerate_target}
+                      COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.py --generate
+                      COMMENT "Regenerating reference output for ${test_name}"
+                      WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    add_dependencies(${regenerate_target} ${test_name})
+    add_dependencies(benchmark_test_regenerate ${regenerate_target})
+endfunction()
+add_custom_target(benchmark_test_regenerate)
+configure_file(test_framework.py.in test_framework.py @ONLY)
+add_benchmark_test(blas)
+add_benchmark_test(conversion)
+add_benchmark_test(matrix_statistics)
+add_benchmark_test(preconditioner)
+add_benchmark_test(solver)
+add_benchmark_test(sparse_blas)
+add_benchmark_test(spmv)
+if (GINKGO_BUILD_MPI)
+    add_benchmark_test(multi_vector_distributed)
+    add_benchmark_test(spmv_distributed)
+    add_benchmark_test(solver_distributed)
+endif()

benchmark/test/blas.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import test_framework
+# check that all input modes work:
+# parameter
+test_framework.compare_output(["blas/blas", "-input", '[{"n": 100}]'],
+                              expected_stdout="blas.simple.stdout",
+                              expected_stderr="blas.simple.stderr")
+
+# stdin
+test_framework.compare_output(["blas/blas"],
+                              expected_stdout="blas.simple.stdout",
+                              expected_stderr="blas.simple.stderr",
+                              stdin='[{"n": 100}]')
+
+# file
+test_framework.compare_output(["blas/blas", "-input", str(test_framework.sourcepath / "input.blas.json")],
+                              expected_stdout="blas.simple.stdout",
+                              expected_stderr="blas.simple.stderr",
+                              stdin='[{"n": 100}]')
+
+# profiler annotations
+test_framework.compare_output(["blas/blas", "-input", '[{"n": 100}]', '-profile', '-profiler_hook', 'debug'],
+                              expected_stdout="blas.profile.stdout",
+                              expected_stderr="blas.profile.stderr",
+                              stdin='[{"n": 100}]')

benchmark/test/conversion.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import test_framework
+# check that all input modes work:
+# parameter
+test_framework.compare_output(["conversion/conversion", "-input", '[{"size": 100, "stencil": "7pt"}]', "-formats", "coo,csr"],
+                              expected_stdout="conversion.simple.stdout",
+                              expected_stderr="conversion.simple.stderr")
+
+# stdin
+test_framework.compare_output(["conversion/conversion", "-formats", "coo,csr"],
+                              expected_stdout="conversion.simple.stdout",
+                              expected_stderr="conversion.simple.stderr",
+                              stdin='[{"size": 100, "stencil": "7pt"}]')
+
+# input file
+test_framework.compare_output(["conversion/conversion", "-input", str(test_framework.sourcepath / "input.mtx.json"), "-formats", "coo,csr"],
+                              expected_stdout="conversion.simple.stdout",
+                              expected_stderr="conversion.simple.stderr")
+
+# check that all conversions work
+test_framework.compare_output(["conversion/conversion", "-input", '[{"size": 100, "stencil": "7pt"}]', "-formats", "coo,csr,ell,sellp,hybrid"],
+                              expected_stdout="conversion.all.stdout",
+                              expected_stderr="conversion.all.stderr")
+
+# profiler annotations
+test_framework.compare_output(["conversion/conversion", "-input", '[{"size": 100, "stencil": "7pt"}]', "-formats", "coo,csr", '-profile', '-profiler_hook', 'debug'],
+                              expected_stdout="conversion.profile.stdout",
+                              expected_stderr="conversion.profile.stderr")

benchmark/test/input.blas.json

@@ -0,0 +1,5 @@
+[
+    {
+        "n": 100
+    }
+]

benchmark/test/input.distributed_mtx.json

@@ -0,0 +1,7 @@
+[
+    {
+        "size": 100,
+        "stencil": "7pt",
+        "comm_pattern": "stencil"
+    }
+]

benchmark/test/input.distributed_solver.json

@@ -0,0 +1,10 @@
+[
+    {
+        "size": 100,
+        "stencil": "7pt",
+        "comm_pattern": "stencil",
+        "optimal": {
+            "spmv": "csr-csr"
+        }
+    }
+]

benchmark/test/input.mtx.json

@@ -0,0 +1,6 @@
+[
+    {
+        "size": 100,
+        "stencil": "7pt"
+    }
+]

benchmark/test/input.solver.json

@@ -0,0 +1,9 @@
+[
+    {
+        "size": 100,
+        "stencil": "7pt",
+        "optimal": {
+            "spmv": "csr"
+        }
+    }
+]

benchmark/test/matrix_statistics.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+import test_framework
+# check that all input modes work:
+# parameter
+test_framework.compare_output(["matrix_statistics/matrix_statistics", "-input", '[{"size": 100, "stencil": "7pt"}]'],
+                              expected_stdout="matrix_statistics.simple.stdout",
+                              expected_stderr="matrix_statistics.simple.stderr")
+
+# stdin
+test_framework.compare_output(["matrix_statistics/matrix_statistics"],
+                              expected_stdout="matrix_statistics.simple.stdout",
+                              expected_stderr="matrix_statistics.simple.stderr",
+                              stdin='[{"size": 100, "stencil": "7pt"}]')
+
+# input file
+test_framework.compare_output(["matrix_statistics/matrix_statistics", "-input", '[{"size": 100, "stencil": "7pt"}]'],
+                              expected_stdout="matrix_statistics.simple.stdout",
+                              expected_stderr="matrix_statistics.simple.stderr")

benchmark/test/multi_vector_distributed.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import test_framework
+base_flags = ["blas/distributed/multi_vector_distributed"]
+# check that all input modes work:
+# parameter
+test_framework.compare_output_distributed(base_flags + ["-input", '[{"n": 100}]'],
+                                          expected_stdout="multi_vector_distributed.simple.stdout",
+                                          expected_stderr="multi_vector_distributed.simple.stderr",
+                                          num_procs=3)
+
+# stdin
+test_framework.compare_output_distributed(base_flags,
+                                          expected_stdout="multi_vector_distributed.simple.stdout",
+                                          expected_stderr="multi_vector_distributed.simple.stderr",
+                                          stdin='[{"n": 100}]',
+                                          num_procs=3)
+
+# file
+test_framework.compare_output_distributed(base_flags + ["-input", str(test_framework.sourcepath / "input.blas.json")],
+                                          expected_stdout="multi_vector_distributed.simple.stdout",
+                                          expected_stderr="multi_vector_distributed.simple.stderr",
+                                          stdin='[{"n": 100}]',
+                                          num_procs=3)
+
+# profiler annotations
+test_framework.compare_output_distributed(base_flags + ["-input", '[{"n": 100}]', '-profile', '-profiler_hook', 'debug'],
+                                          expected_stdout="multi_vector_distributed.profile.stdout",
+                                          expected_stderr="multi_vector_distributed.profile.stderr",
+                                          stdin='[{"n": 100}]',
+                                          num_procs=3)

benchmark/test/preconditioner.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import test_framework
+# check that all input modes work:
+# parameter
+test_framework.compare_output(["preconditioner/preconditioner", "-input", '[{"size": 100, "stencil": "7pt"}]'],
+                              expected_stdout="preconditioner.simple.stdout",
+                              expected_stderr="preconditioner.simple.stderr")
+
+# stdin
+test_framework.compare_output(["preconditioner/preconditioner"],
+                              expected_stdout="preconditioner.simple.stdout",
+                              expected_stderr="preconditioner.simple.stderr",
+                              stdin='[{"size": 100, "stencil": "7pt"}]')
+
+# input file
+test_framework.compare_output(["preconditioner/preconditioner", "-input", str(test_framework.sourcepath / "input.mtx.json")],
+                              expected_stdout="preconditioner.simple.stdout",
+                              expected_stderr="preconditioner.simple.stderr")
+
+# profiler annotations
+test_framework.compare_output(["preconditioner/preconditioner", "-input", '[{"size": 100, "stencil": "7pt"}]', '-profile', '-profiler_hook', 'debug'],
+                              expected_stdout="preconditioner.profile.stdout",
+                              expected_stderr="preconditioner.profile.stderr")

benchmark/test/reference/blas.profile.stderr

@@ -0,0 +1,130 @@
+This is Ginkgo 1.5.0 (develop)
+    running with core module 1.5.0 (develop)
+Running on reference(0)
+Running with 0 warm iterations and 1 running iterations
+The random seed for right hand sides is 42
+The operations are copy,axpy,scalRunning test case
+{
+    "n": 100,
+    "blas": {}
+}
+DEBUG: begin n = 100 
+DEBUG: begin copy
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::copy
+DEBUG: end   dense::copy
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: end   copy
+Current state:
+[
+    {
+        "n": 100,
+        "blas": {
+            "copy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            }
+        }
+    }
+]
+DEBUG: begin axpy
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::add_scaled
+DEBUG: end   dense::add_scaled
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: end   axpy
+Current state:
+[
+    {
+        "n": 100,
+        "blas": {
+            "copy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            },
+            "axpy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            }
+        }
+    }
+]
+DEBUG: begin scal
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin allocate
+DEBUG: end   allocate
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::fill
+DEBUG: end   dense::fill
+DEBUG: begin dense::scale
+DEBUG: end   dense::scale
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: begin free
+DEBUG: end   free
+DEBUG: end   scal
+Current state:
+[
+    {
+        "n": 100,
+        "blas": {
+            "copy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            },
+            "axpy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            },
+            "scal": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            }
+        }
+    }
+]
+DEBUG: end   n = 100 

benchmark/test/reference/blas.profile.stdout

@@ -0,0 +1,29 @@
+
+[
+    {
+        "n": 100,
+        "blas": {
+            "copy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            },
+            "axpy": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            },
+            "scal": {
+                "time": 1.0,
+                "flops": 1.0,
+                "bandwidth": 1.0,
+                "repetitions": 1,
+                "completed": true
+            }
+        }
+    }
+]