diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_build/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_build/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..d67c52a7f68f --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_build/AGGREGATE_analysis.txt @@ -0,0 +1,172 @@ +======================================================================================================================== +TIMING ANALYSIS SUMMARY +======================================================================================================================== + +TIMING BY PURPOSE +------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 1,152,118.78 100.0% + BUILD: solution generation & compile 1,104,771.86 95.9% 95.9% + Solution generation & build (Python) 1,104,771.86 100.0% 95.9% + python_kernel_compilation 1,005 807,165.44 803.15 73.1% 70.1% + python_kernel_codegen 123 268,182.72 2,180.35 33.2% 23.3% + python_kernel_write_assemble 441 173,421.33 393.25 21.5% 15.1% + python_kernel_validate 441 125,911.83 285.51 15.6% 10.9% + python_kernel_bench_postprocess 441 122,727.30 278.29 15.2% 10.7% + python_benchpost_lib_construction 441 77,770.74 176.35 63.4% 6.8% + python_benchpost_library_write 441 32,602.71 73.93 26.6% 2.8% + python_benchpost_naming 441 12,213.89 27.70 10.0% 1.1% + python_benchpost_client_config 441 112.12 0.25 0.1% 0.0% + (untracked) 27.83 0.0% 0.0% + python_kernel_build_src_co_wait 441 47,743.03 108.26 5.9% 4.1% + python_kernel_setup 441 31,525.89 71.49 3.9% 2.7% + python_kernel_build_co 441 20,667.26 46.86 2.6% 1.8% + python_kernel_write_helpers 441 6,792.76 15.40 0.8% 0.6% + python_kernel_bench_setup 441 4,417.09 10.02 0.5% 0.4% + (untracked) 5,776.25 0.7% 0.5% + python_solution_generation 441 297,606.41 674.84 26.9% 25.8% + python_solgen_forked_solutions 441 296,425.93 672.17 99.6% 25.7% + python_solgen_custom_kernels 441 1,168.99 2.65 0.4% 0.1% + python_solgen_fork_permutations 441 3.18 0.01 0.0% 0.0% + (untracked) 8.32 0.0% 0.0% + SHARED: cache & solution files 32,643.25 2.8% 2.8% + python_write_solutions 441 20,547.22 46.59 62.9% 1.8% + python_wsol_dump 441 19,990.69 45.33 97.3% 1.7% + python_wsol_prepare 441 453.03 1.03 2.2% 0.0% + python_wsol_prepare_nocache 441 451.75 1.02 99.7% 0.0% + (untracked) 1.28 0.3% 0.0% + python_wsol_header 441 9.24 0.02 0.0% 0.0% + (untracked) 94.25 0.5% 0.0% + python_solution_indexing 441 11,484.12 26.04 35.2% 1.0% + python_write_cache 441 611.27 1.39 1.9% 0.1% + python_cache_check 441 0.64 0.00 0.0% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 14,703.68 1.3% 1.3% + (untracked in python_benchmark_problems) 14,703.68 100.0% 1.3% + +TIMING BY TIMER NESTING (legacy view) +------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------ + python_benchmark_problems 123 1,152,118.78 9,366.82 100.0% + -------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 1,005 807,165.44 803.15 70.1% 70.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 123 268,182.72 2,180.35 33.2% 23.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 441 173,421.33 393.25 21.5% 15.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 441 125,911.83 285.51 15.6% 10.9% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 441 122,727.30 278.29 15.2% 10.7% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 441 77,770.74 176.35 63.4% 6.8% + python_benchpost_library_write 441 32,602.71 73.93 26.6% 2.8% + python_benchpost_naming 441 12,213.89 27.70 10.0% 1.1% + python_benchpost_client_config 441 112.12 0.25 0.1% 0.0% + (untracked) 27.83 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_src_co_wait 441 47,743.03 108.26 5.9% 4.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 441 31,525.89 71.49 3.9% 2.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 441 20,667.26 46.86 2.6% 1.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 441 6,792.76 15.40 0.8% 0.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 5,776.25 0.7% 0.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 441 4,417.09 10.02 0.5% 0.4% + -------------------------------------------------------------------------------------------------------------------- + python_solution_generation 441 297,606.41 674.84 25.8% 25.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_forked_solutions 441 296,425.93 672.17 99.6% 25.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_custom_kernels 441 1,168.99 2.65 0.4% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 8.32 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_fork_permutations 441 3.18 0.01 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------- + python_write_solutions 441 20,547.22 46.59 1.8% 1.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 441 19,990.69 45.33 97.3% 1.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 441 453.03 1.03 2.2% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_nocache 441 451.75 1.02 99.7% 0.0% + (untracked) 1.28 0.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 94.25 0.5% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 441 9.24 0.02 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------- + (untracked) 14,703.67 1.3% 1.3% + -------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 441 11,484.12 26.04 1.0% 1.0% + -------------------------------------------------------------------------------------------------------------------- + python_write_cache 441 611.27 1.39 0.1% 0.1% + -------------------------------------------------------------------------------------------------------------------- + python_cache_check 441 0.64 0.00 0.0% 0.0% + + Overlapping Totals (wrap or overlap other timers; not summed into the tree) + ---------------------------------------------------------------------------------------------------------------------- + python_kernel_build_src_co 441 185,511.28 420.66 16.1% + python_kernel_build_src_co.setup 441 224.67 0.51 0.0% + python_kernel_build_src_co.cache_check 441 5,995.91 13.60 0.5% + python_kernel_build_src_co.pch 97 4,780.47 49.28 0.4% + python_kernel_build_src_co.compile 97 166,252.19 1,713.94 14.4% + python_kernel_build_src_co.unbundle 97 878.00 9.05 0.1% + python_kernel_build_src_co.move 97 27.55 0.28 0.0% + python_kernel_build_src_co.cache_populate 97 248.95 2.57 0.0% + +------------------------------------------------------------------------------------------------------------------------ +HIERARCHY VALIDATION: OK (all child phases fit within their parents) + +TIME BREAKDOWN (visual, % of wall clock) +------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 100.0% ################################################## + BUILD: solution generation & compile 95.9% ############################################### + Solution generation & build (Python) 95.9% ############################################### + python_kernel_compilation 70.1% ################################### + python_kernel_codegen 23.3% ########### + python_kernel_write_assemble 15.1% ####### + python_kernel_validate 10.9% ##### + python_kernel_bench_postprocess 10.7% ##### + python_benchpost_lib_construction 6.8% ### + python_benchpost_library_write 2.8% # + python_benchpost_naming 1.1% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_build_src_co_wait 4.1% ## + python_kernel_setup 2.7% # + python_kernel_build_co 1.8% + python_kernel_write_helpers 0.6% + python_kernel_bench_setup 0.4% + (untracked) 0.5% + python_solution_generation 25.8% ############ + python_solgen_forked_solutions 25.7% ############ + python_solgen_custom_kernels 0.1% + python_solgen_fork_permutations 0.0% + (untracked) 0.0% + SHARED: cache & solution files 2.8% # + python_write_solutions 1.8% + python_wsol_dump 1.7% + python_wsol_prepare 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + python_solution_indexing 1.0% + python_write_cache 0.1% + python_cache_check 0.0% + UNATTRIBUTED (untracked time & timer overhead) 1.3% + (untracked in python_benchmark_problems) 1.3% + +KEY INSIGHTS (% of wall clock) +------------------------------------------------------------------------------------------------------------------------ + Solution generation & build (Python) 1104.77s (95.9%) + SHARED: cache & solution files 32.64s (2.8%) + UNATTRIBUTED (untracked time & timer overhead) 14.70s (1.3%) + + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_run/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_run/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..8841b9e0d152 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/post_run/AGGREGATE_analysis.txt @@ -0,0 +1,585 @@ +==================================================================================================================================== +TIMING ANALYSIS SUMMARY +==================================================================================================================================== + +TIMING BY PURPOSE +------------------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 828,655.65 100.0% + RUN: benchmark client 718,919.62 86.8% 86.8% + Client launch (startup, setup & teardown) 468,439.79 65.2% 56.5% + listener_setup 441 179,268.92 406.51 38.3% 21.6% + hip_initialization 441 167,901.69 380.73 35.8% 20.3% + process_exit 441 54,706.34 124.05 11.7% 6.6% + library_loading 441 17,247.98 39.11 3.7% 2.1% + data_init_setup 441 15,082.37 34.20 3.2% 1.8% + code_object_loading 441 13,085.11 29.67 2.8% 1.6% + main_teardown_locals 441 11,335.07 25.70 2.4% 1.4% + process_startup 441 8,596.68 19.49 1.8% 1.0% + lazy_loading_init 441 793.21 1.80 0.2% 0.1% + main_parse_args 441 245.05 0.56 0.1% 0.0% + main_problem_factory 441 92.29 0.21 0.0% 0.0% + (untracked) 92.29 100.0% 0.0% + python_client_write_run_script 441 54.17 0.12 0.0% 0.0% + reporter_setup 441 23.17 0.05 0.0% 0.0% + solution_iterator_setup 441 7.74 0.02 0.0% 0.0% + Kernel launch & execution 67,792.53 9.4% 8.2% + warmup_first_sync 1,229,030 50,716.93 0.04 74.8% 6.1% + warmup_runs 2,458,060 11,484.24 0.00 16.9% 1.4% + kernel_solving 1,229,034 3,365.17 0.00 5.0% 0.4% + benchmark_runs 1,229,034 1,206.77 0.00 1.8% 0.1% + benchmark_validate_enqueues 602 800.92 1.33 66.4% 0.1% + validate_enqueues_listener.HardwareMonitorListener 602 358.72 0.60 44.8% 0.0% + validate_enqueues_listener.ReferenceValidator 602 220.51 0.37 27.5% 0.0% + validate_enqueues_listener.BenchmarkTimer 602 199.42 0.33 24.9% 0.0% + benchmark_sleep 601 163.57 0.27 82.0% 0.0% + benchmark_event_sync 219 35.49 0.16 17.8% 0.0% + benchmark_event_elapsed_time 219 0.20 0.00 0.1% 0.0% + benchmark_event_destroy 219 0.16 0.00 0.1% 0.0% + validate_enqueues_listener.SynchronizerValidator 602 14.01 0.02 1.7% 0.0% + validate_enqueues_listener.ProgressListener 602 7.28 0.01 0.9% 0.0% + validate_enqueues_listener.MetaResultReporter 602 0.03 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 602 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 602 0.02 0.00 0.0% 0.0% + (untracked) 0.91 0.1% 0.0% + gpu_kernel_execution 602 305.37 0.51 25.3% 0.0% + benchmark_post_syncs 1,229,034 53.04 0.00 4.4% 0.0% + benchmark_pre_syncs 1,229,034 46.49 0.00 3.9% 0.0% + benchmark_events_setup 602 0.68 0.00 0.1% 0.0% + benchmark_events_teardown 602 0.28 0.00 0.0% 0.0% + warmup_events_setup 1,229,034 611.83 0.00 0.9% 0.1% + run_iteration_sync 1,229,034 372.15 0.00 0.5% 0.0% + validate_warmups_listener.BenchmarkTimer 1,229,030 35.42 0.00 0.1% 0.0% + validate_gpu_sync 598 0.44 0.00 1.2% 0.0% + (untracked) 34.98 98.8% 0.0% + Validation 66,752.37 9.3% 8.1% + validate_warmups_listener.ReferenceValidator 1,229,030 61,018.96 0.05 91.4% 7.4% + validate_element_comparison 1,042,067 32,369.88 0.03 53.0% 3.9% + validate_gpu_readback 1,042,067 24,904.42 0.02 40.8% 3.0% + validate_result_buffer_alloc 1,042,067 3,253.38 0.00 5.3% 0.4% + validate_inputs_copy 1,036,487 134.21 0.00 0.2% 0.0% + validate_setup 1,037,703 30.65 0.00 0.1% 0.0% + validate_report 1,042,067 30.60 0.00 0.1% 0.0% + (untracked) 295.81 0.5% 0.0% + synchronizer_check 1,229,034 5,522.58 0.00 8.3% 0.7% + synchronizer_reduce 253,709 5,354.20 0.02 97.0% 0.6% + synchronizer_inactive 975,349 23.65 0.00 0.4% 0.0% + (untracked) 144.73 2.6% 0.0% + validate_warmups_listener.MetaResultReporter 1,229,030 47.93 0.00 0.1% 0.0% + validate_warmups_listener.HardwareMonitorListener 1,229,030 34.44 0.00 0.1% 0.0% + validate_warmups_listener.ProgressListener 1,229,030 32.62 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1,229,030 32.38 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1,229,030 32.12 0.00 0.0% 0.0% + (untracked) 32.12 100.0% 0.0% + validate_warmups_listener.DataInitialization 1,229,030 31.34 0.00 0.0% 0.0% + Input data preparation 53,726.75 7.5% 6.5% + gpu_input_reset 1,229,034 31,704.99 0.03 59.0% 3.8% + gpu_input_reset_batched_inputs 658,043 19,665.29 0.03 62.0% 2.4% + gpu_input_reset_copy_inputs_d2d 658,043 10,024.05 0.02 31.6% 1.2% + gpu_input_reset_fast_reset_output 570,863 1,625.23 0.00 5.1% 0.2% + gpu_input_reset_convert_to_inputs 658,043 83.75 0.00 0.3% 0.0% + gpu_input_reset_copy_valid_to_gpu 137,906 59.24 0.00 0.2% 0.0% + gpu_input_reset_slow_path 658,043 16.04 0.00 0.1% 0.0% + gpu_input_reset_copy_swizzled 13,132 2.45 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.04 0.02 0.0% 0.0% + gpu_input_reset_constant_inputs 5 0.02 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 5 0.01 0.00 0.0% 0.0% + (untracked) 228.87 0.7% 0.0% + cpu_data_init 4,029 15,135.73 3.76 28.2% 1.8% + gpu_input_preparation 4,035 6,761.73 1.68 12.6% 0.8% + gpu_input_prep_copy_valid_to_gpu 1,797 4,948.50 2.75 73.2% 0.6% + gpu_input_prep_copy_swizzled 98 973.72 9.94 14.4% 0.1% + gpu_input_prep_init_cpu_inputs 3 573.60 191.20 8.5% 0.1% + gpu_input_prep_batched_inputs 2,903 169.01 0.06 2.5% 0.0% + gpu_input_prep_copy_inputs_d2d 2,903 76.10 0.03 1.1% 0.0% + gpu_input_prep_fast_reset_output 1,119 9.77 0.01 0.1% 0.0% + gpu_input_prep_convert_to_inputs 2,903 1.76 0.00 0.0% 0.0% + gpu_input_prep_slow_path 2,903 0.14 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 4 0.04 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + (untracked) 9.07 0.1% 0.0% + rotating_buffer_preparation 4,035 82.06 0.02 0.2% 0.0% + pre_solution_listener.DataInitialization 1,286,411 42.24 0.00 0.1% 0.0% + pre_solution_mx_check 1,286,411 33.65 0.00 79.7% 0.0% + (untracked) 8.58 20.3% 0.0% + Reference GEMM (CPU) 38,352.93 5.3% 4.6% + cpu_reference_gemm 4,029 38,288.38 9.50 99.8% 4.6% + solve_cpu_fast 3,503 37,422.80 10.68 97.7% 4.5% + solve_cpu_slow 585 858.06 1.47 2.2% 0.1% + (untracked) 7.52 0.0% 0.0% + pre_solution_listener.ReferenceValidator 1,286,411 64.55 0.00 0.2% 0.0% + pre_solution_ref_skipped 1,286,230 31.28 0.00 48.5% 0.0% + (untracked) 33.28 51.5% 0.0% + Per-solution bookkeeping & reporting 9,263.67 1.3% 1.1% + run_current_solution_check 1,286,411 1,592.51 0.00 17.2% 0.2% + post_solution_reporting 1,286,411 1,468.29 0.00 15.8% 0.2% + pre_solution_listener.BenchmarkTimer 1,286,411 1,459.57 0.00 15.8% 0.2% + post_solution_log 1,286,411 1,050.71 0.00 11.3% 0.1% + pre_solution_listener.AllSolutionsIterator 1,286,411 981.53 0.00 10.6% 0.1% + pre_solution_listener.MetaResultReporter 1,286,411 607.95 0.00 6.6% 0.1% + solution_selection 1,286,411 586.47 0.00 6.3% 0.1% + pre_solution_listener.ProgressListener 1,286,411 556.61 0.00 6.0% 0.1% + post_solution_validation 1,286,411 307.05 0.00 3.3% 0.0% + pre_solution_listener.SynchronizerValidator 1,286,411 211.26 0.00 2.3% 0.0% + run_iteration_teardown 1,229,034 195.86 0.00 2.1% 0.0% + post_problem 4,035 65.20 0.02 0.7% 0.0% + solution_teardown 1,286,411 43.35 0.00 0.5% 0.0% + pre_solution_listener.HardwareMonitorListener 1,286,411 34.15 0.00 0.4% 0.0% + post_solution_perf_calc 1,286,411 33.63 0.00 0.4% 0.0% + post_solution_sol_advance 1,286,411 32.80 0.00 0.4% 0.0% + post_solution_perf_reset 1,286,411 31.38 0.00 0.3% 0.0% + problem_report_index 4,035 4.79 0.00 0.1% 0.0% + finalize_report 441 0.29 0.00 0.0% 0.0% + problem_teardown 4,035 0.25 0.00 0.0% 0.0% + post_solution_result_file 335 0.01 0.00 0.0% 0.0% + Library logic (Python) 8,379.03 1.2% 1.0% + python_logic_write 5 7,278.92 1,455.78 86.9% 0.9% + python_logic_parse_solutions 5 987.41 197.48 11.8% 0.1% + python_logic_analyze 5 112.70 22.54 1.3% 0.0% + Library client (Python) 6,212.54 0.9% 0.7% + python_library_client_create_library 1 4,131.51 4,131.51 66.5% 0.5% + python_library_client_execution 1 2,070.88 2,070.88 33.3% 0.2% + python_library_client_subprocess 1 2,070.26 2,070.26 100.0% 0.2% + python_library_client_write_run_script 1 0.08 0.08 0.0% 0.0% + (untracked) 0.54 0.0% 0.0% + python_library_client_write_configs 1 8.63 8.63 0.1% 0.0% + python_library_client_setup 1 1.52 1.52 0.0% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 107,970.48 13.0% 13.0% + flush_timing_buffer 443 89,828.91 202.77 83.2% 10.8% + main_timing_calibration 441 10,990.36 24.92 10.2% 1.3% + timing_overhead 441 3,700.16 8.39 3.4% 0.4% + (timer overhead subtracted from parent timers) 3,451.04 3.2% 0.4% + SHARED: cache & solution files 965.91 0.1% 0.1% + python_cache_check 441 847.49 1.92 87.7% 0.1% + python_write_solutions 441 118.42 0.27 12.3% 0.0% + python_wsol_prepare 441 85.38 0.19 72.1% 0.0% + python_wsol_prepare_cache 441 82.85 0.19 97.0% 0.0% + (untracked) 2.54 3.0% 0.0% + python_wsol_header 441 26.45 0.06 22.3% 0.0% + (untracked) 6.58 5.6% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 799.64 0.1% 0.1% + (untracked in python_benchmark_problems) 3,199.25 400.1% 0.4% + (untracked in validate_warmups) 336.05 42.0% 0.0% + (untracked in pre_solution) 262.48 32.8% 0.0% + (untracked in pre_problem) 39.53 4.9% 0.0% + (untracked in python_library_logic) 5.15 0.6% 0.0% + (untracked in python_client_writer) 0.93 0.1% 0.0% + (children exceed benchmark_runs) -16.61 -2.1% -0.0% + (children exceed validate_enqueues_listener.BenchmarkTimer) -35.42 -4.4% -0.0% + (children exceed post_solution) -52.43 -6.6% -0.0% + (children exceed python_client_execution) -2,939.28 -367.6% -0.4% + +TIMING BY TIMER NESTING (legacy view) +------------------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------------------ + python_benchmark_problems 123 810,606.95 6,590.30 97.8% + -------------------------------------------------------------------------------------------------------------------------------- + python_client_execution 441 806,441.80 1,828.67 99.5% 97.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 479,430.15 59.5% 57.9% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + listener_setup 441 179,268.92 406.51 37.4% 21.6% + hip_initialization 441 167,901.69 380.73 35.0% 20.3% + process_exit 441 54,706.34 124.05 11.4% 6.6% + library_loading 441 17,247.98 39.11 3.6% 2.1% + data_init_setup 441 15,082.37 34.20 3.1% 1.8% + code_object_loading 441 13,085.11 29.67 2.7% 1.6% + main_teardown_locals 441 11,335.07 25.70 2.4% 1.4% + main_timing_calibration 441 10,990.36 24.92 2.3% 1.3% + process_startup 441 8,596.68 19.49 1.8% 1.0% + lazy_loading_init 441 793.21 1.80 0.2% 0.1% + main_parse_args 441 245.05 0.56 0.1% 0.0% + main_problem_factory 441 92.29 0.21 0.0% 0.0% + (untracked) 92.29 100.0% 0.0% + python_client_write_run_script 441 54.17 0.12 0.0% 0.0% + reporter_setup 441 23.17 0.05 0.0% 0.0% + solution_iterator_setup 441 7.74 0.02 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 93,529.07 11.6% 11.3% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 443 89,828.91 202.77 96.0% 10.8% + timing_overhead 441 3,700.16 8.39 4.0% 0.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 92,012.42 11.4% 11.1% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_problem 4,035 53,463.64 13.25 58.1% 6.5% + cpu_reference_gemm 4,029 38,288.38 9.50 71.6% 4.6% + solve_cpu_fast 3,503 37,422.80 10.68 97.7% 4.5% + solve_cpu_slow 585 858.06 1.47 2.2% 0.1% + (untracked) 7.52 0.0% 0.0% + cpu_data_init 4,029 15,135.73 3.76 28.3% 1.8% + (untracked) 39.53 0.1% 0.0% + gpu_input_reset 1,229,034 31,704.99 0.03 34.5% 3.8% + gpu_input_reset_batched_inputs 658,043 19,665.29 0.03 62.0% 2.4% + gpu_input_reset_copy_inputs_d2d 658,043 10,024.05 0.02 31.6% 1.2% + gpu_input_reset_fast_reset_output 570,863 1,625.23 0.00 5.1% 0.2% + (untracked) 228.87 0.7% 0.0% + gpu_input_reset_convert_to_inputs 658,043 83.75 0.00 0.3% 0.0% + gpu_input_reset_copy_valid_to_gpu 137,906 59.24 0.00 0.2% 0.0% + gpu_input_reset_slow_path 658,043 16.04 0.00 0.1% 0.0% + gpu_input_reset_copy_swizzled 13,132 2.45 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.04 0.02 0.0% 0.0% + gpu_input_reset_constant_inputs 5 0.02 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 5 0.01 0.00 0.0% 0.0% + gpu_input_preparation 4,035 6,761.73 1.68 7.3% 0.8% + gpu_input_prep_copy_valid_to_gpu 1,797 4,948.50 2.75 73.2% 0.6% + gpu_input_prep_copy_swizzled 98 973.72 9.94 14.4% 0.1% + gpu_input_prep_init_cpu_inputs 3 573.60 191.20 8.5% 0.1% + gpu_input_prep_batched_inputs 2,903 169.01 0.06 2.5% 0.0% + gpu_input_prep_copy_inputs_d2d 2,903 76.10 0.03 1.1% 0.0% + gpu_input_prep_fast_reset_output 1,119 9.77 0.01 0.1% 0.0% + (untracked) 9.07 0.1% 0.0% + gpu_input_prep_convert_to_inputs 2,903 1.76 0.00 0.0% 0.0% + gpu_input_prep_slow_path 2,903 0.14 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 4 0.04 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + rotating_buffer_preparation 4,035 82.06 0.02 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 77,285.59 9.6% 9.3% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + warmup_first_sync 1,229,030 50,716.93 0.04 65.6% 6.1% + warmup_runs 2,458,060 11,484.24 0.00 14.9% 1.4% + pre_solution 1,286,411 4,220.34 0.00 5.5% 0.5% + pre_solution_listener.BenchmarkTimer 1,286,411 1,459.57 0.00 34.6% 0.2% + pre_solution_listener.AllSolutionsIterator 1,286,411 981.53 0.00 23.3% 0.1% + pre_solution_listener.MetaResultReporter 1,286,411 607.95 0.00 14.4% 0.1% + pre_solution_listener.ProgressListener 1,286,411 556.61 0.00 13.2% 0.1% + (untracked) 262.48 6.2% 0.0% + pre_solution_listener.SynchronizerValidator 1,286,411 211.26 0.00 5.0% 0.0% + pre_solution_listener.ReferenceValidator 1,286,411 64.55 0.00 1.5% 0.0% + (untracked) 33.28 51.5% 0.0% + pre_solution_ref_skipped 1,286,230 31.28 0.00 48.5% 0.0% + pre_solution_listener.DataInitialization 1,286,411 42.24 0.00 1.0% 0.0% + pre_solution_mx_check 1,286,411 33.65 0.00 79.7% 0.0% + (untracked) 8.58 20.3% 0.0% + pre_solution_listener.HardwareMonitorListener 1,286,411 34.15 0.00 0.8% 0.0% + kernel_solving 1,229,034 3,365.17 0.00 4.4% 0.4% + post_solution 1,286,411 2,871.44 0.00 3.7% 0.3% + post_solution_reporting 1,286,411 1,468.29 0.00 51.1% 0.2% + post_solution_log 1,286,411 1,050.71 0.00 36.6% 0.1% + post_solution_validation 1,286,411 307.05 0.00 10.7% 0.0% + post_solution_perf_calc 1,286,411 33.63 0.00 1.2% 0.0% + post_solution_sol_advance 1,286,411 32.80 0.00 1.1% 0.0% + post_solution_perf_reset 1,286,411 31.38 0.00 1.1% 0.0% + post_solution_result_file 335 0.01 0.00 0.0% 0.0% + run_current_solution_check 1,286,411 1,592.51 0.00 2.1% 0.2% + benchmark_runs 1,229,034 1,154.74 0.00 1.5% 0.1% + benchmark_validate_enqueues 602 765.50 1.27 66.3% 0.1% + validate_enqueues_listener.HardwareMonitorListener 602 358.72 0.60 46.9% 0.0% + validate_enqueues_listener.ReferenceValidator 602 220.51 0.37 28.8% 0.0% + validate_enqueues_listener.BenchmarkTimer 602 164.00 0.27 21.4% 0.0% + benchmark_sleep 601 163.57 0.27 99.7% 0.0% + benchmark_event_sync 219 35.49 0.16 21.6% 0.0% + benchmark_event_elapsed_time 219 0.20 0.00 0.1% 0.0% + benchmark_event_destroy 219 0.16 0.00 0.1% 0.0% + validate_enqueues_listener.SynchronizerValidator 602 14.01 0.02 1.8% 0.0% + validate_enqueues_listener.ProgressListener 602 7.28 0.01 1.0% 0.0% + (untracked) 0.91 0.1% 0.0% + validate_enqueues_listener.MetaResultReporter 602 0.03 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 602 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 602 0.02 0.00 0.0% 0.0% + gpu_kernel_execution 602 305.37 0.51 26.4% 0.0% + benchmark_post_syncs 1,229,034 53.04 0.00 4.6% 0.0% + benchmark_pre_syncs 1,229,034 46.49 0.00 4.0% 0.0% + benchmark_events_setup 602 0.68 0.00 0.1% 0.0% + benchmark_events_teardown 602 0.28 0.00 0.0% 0.0% + warmup_events_setup 1,229,034 611.83 0.00 0.8% 0.1% + solution_selection 1,286,411 586.47 0.00 0.8% 0.1% + run_iteration_sync 1,229,034 372.15 0.00 0.5% 0.0% + run_iteration_teardown 1,229,034 195.86 0.00 0.3% 0.0% + post_problem 4,035 65.20 0.02 0.1% 0.0% + solution_teardown 1,286,411 43.35 0.00 0.1% 0.0% + problem_report_index 4,035 4.79 0.00 0.0% 0.0% + finalize_report 441 0.29 0.00 0.0% 0.0% + problem_teardown 4,035 0.25 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 67,123.84 8.3% 8.1% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 1,229,030 61,601.26 0.05 91.8% 7.4% + validate_warmups_listener.ReferenceValidator 1,229,030 61,018.96 0.05 99.1% 7.4% + validate_element_comparison 1,042,067 32,369.88 0.03 53.0% 3.9% + validate_gpu_readback 1,042,067 24,904.42 0.02 40.8% 3.0% + validate_result_buffer_alloc 1,042,067 3,253.38 0.00 5.3% 0.4% + (untracked) 295.81 0.5% 0.0% + validate_inputs_copy 1,036,487 134.21 0.00 0.2% 0.0% + validate_setup 1,037,703 30.65 0.00 0.1% 0.0% + validate_report 1,042,067 30.60 0.00 0.1% 0.0% + (untracked) 336.05 0.5% 0.0% + validate_warmups_listener.MetaResultReporter 1,229,030 47.93 0.00 0.1% 0.0% + validate_warmups_listener.BenchmarkTimer 1,229,030 35.42 0.00 0.1% 0.0% + (untracked) 34.98 98.8% 0.0% + validate_gpu_sync 598 0.44 0.00 1.2% 0.0% + validate_warmups_listener.HardwareMonitorListener 1,229,030 34.44 0.00 0.1% 0.0% + validate_warmups_listener.ProgressListener 1,229,030 32.62 0.00 0.1% 0.0% + validate_warmups_listener.AllSolutionsIterator 1,229,030 32.38 0.00 0.1% 0.0% + validate_warmups_listener.SynchronizerValidator 1,229,030 32.12 0.00 0.1% 0.0% + (untracked) 32.12 100.0% 0.0% + validate_warmups_listener.DataInitialization 1,229,030 31.34 0.00 0.1% 0.0% + synchronizer_check 1,229,034 5,522.58 0.00 8.2% 0.7% + synchronizer_reduce 253,709 5,354.20 0.02 97.0% 0.6% + (untracked) 144.73 2.6% 0.0% + synchronizer_inactive 975,349 23.65 0.00 0.4% 0.0% + -------------------------------------------------------------------------------------------------------------------------------- + (untracked) 3,199.25 0.4% 0.4% + -------------------------------------------------------------------------------------------------------------------------------- + python_cache_check 441 847.49 1.92 0.1% 0.1% + -------------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 441 118.42 0.27 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 441 85.38 0.19 72.1% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_cache 441 82.85 0.19 97.0% 0.0% + (untracked) 2.54 3.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 441 26.45 0.06 22.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 6.58 5.6% 0.0% + ================================================================================================================================== + python_library_logic 5 8,384.18 1,676.84 1.0% + -------------------------------------------------------------------------------------------------------------------------------- + python_logic_write 5 7,278.92 1,455.78 86.8% 0.9% + -------------------------------------------------------------------------------------------------------------------------------- + python_logic_parse_solutions 5 987.41 197.48 11.8% 0.1% + -------------------------------------------------------------------------------------------------------------------------------- + python_logic_analyze 5 112.70 22.54 1.3% 0.0% + -------------------------------------------------------------------------------------------------------------------------------- + (untracked) 5.15 0.1% 0.0% + ================================================================================================================================== + python_client_writer 1 6,213.47 6,213.47 0.7% + -------------------------------------------------------------------------------------------------------------------------------- + python_library_client_create_library 1 4,131.51 4,131.51 66.5% 0.5% + -------------------------------------------------------------------------------------------------------------------------------- + python_library_client_execution 1 2,070.88 2,070.88 33.3% 0.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_subprocess 1 2,070.26 2,070.26 100.0% 0.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 0.54 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_write_run_script 1 0.08 0.08 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------------------- + python_library_client_write_configs 1 8.63 8.63 0.1% 0.0% + -------------------------------------------------------------------------------------------------------------------------------- + python_library_client_setup 1 1.52 1.52 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------------------- + (untracked) 0.93 0.0% 0.0% + + Unknown Categories + ---------------------------------------------------------------------------------------------------------------------------------- + icache_rotate_extra_copies_loading 1 5.86 5.86 0.0% + + Overlapping Totals (wrap or overlap other timers; not summed into the tree) + ---------------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 441 809,527.67 1,835.66 97.7% + main_total 441 645,030.44 1,462.65 77.8% + problem_total 4,035 240,270.61 59.55 29.0% + solution_total 1,286,411 179,822.86 0.14 21.7% + run_iteration_total 1,229,034 168,774.82 0.14 20.4% + validate_total 1,037,703 61,012.13 0.06 7.4% + validate_check_results_typed 1,042,067 60,795.53 0.06 7.3% + +------------------------------------------------------------------------------------------------------------------------------------ +HIERARCHY VALIDATION WARNINGS +------------------------------------------------------------------------------------------------------------------------------------ + WARNING: Child phases of python_client_execution (811,749.69 ms) exceed parent (809,892.69 ms) by 1,856.99 ms (0.2%) + WARNING: Child phases of validate_enqueues_listener.BenchmarkTimer (199.42 ms) exceed parent (164.07 ms) by 35.36 ms (21.5%) + +TIME BREAKDOWN (visual, % of wall clock) +------------------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 100.0% ################################################## + RUN: benchmark client 86.8% ########################################### + Client launch (startup, setup & teardown) 56.5% ############################ + listener_setup 21.6% ########## + hip_initialization 20.3% ########## + process_exit 6.6% ### + library_loading 2.1% # + data_init_setup 1.8% + code_object_loading 1.6% + main_teardown_locals 1.4% + process_startup 1.0% + lazy_loading_init 0.1% + main_parse_args 0.0% + main_problem_factory 0.0% + (untracked) 0.0% + python_client_write_run_script 0.0% + reporter_setup 0.0% + solution_iterator_setup 0.0% + Kernel launch & execution 8.2% #### + warmup_first_sync 6.1% ### + warmup_runs 1.4% + kernel_solving 0.4% + benchmark_runs 0.1% + benchmark_validate_enqueues 0.1% + validate_enqueues_listener.HardwareMonitorListener 0.0% + validate_enqueues_listener.ReferenceValidator 0.0% + validate_enqueues_listener.BenchmarkTimer 0.0% + benchmark_sleep 0.0% + benchmark_event_sync 0.0% + benchmark_event_elapsed_time 0.0% + benchmark_event_destroy 0.0% + validate_enqueues_listener.SynchronizerValidator 0.0% + validate_enqueues_listener.ProgressListener 0.0% + validate_enqueues_listener.MetaResultReporter 0.0% + validate_enqueues_listener.DataInitialization 0.0% + validate_enqueues_listener.AllSolutionsIterator 0.0% + (untracked) 0.0% + gpu_kernel_execution 0.0% + benchmark_post_syncs 0.0% + benchmark_pre_syncs 0.0% + benchmark_events_setup 0.0% + benchmark_events_teardown 0.0% + warmup_events_setup 0.1% + run_iteration_sync 0.0% + validate_warmups_listener.BenchmarkTimer 0.0% + validate_gpu_sync 0.0% + (untracked) 0.0% + Validation 8.1% #### + validate_warmups_listener.ReferenceValidator 7.4% ### + validate_element_comparison 3.9% # + validate_gpu_readback 3.0% # + validate_result_buffer_alloc 0.4% + validate_inputs_copy 0.0% + validate_setup 0.0% + validate_report 0.0% + (untracked) 0.0% + synchronizer_check 0.7% + synchronizer_reduce 0.6% + synchronizer_inactive 0.0% + (untracked) 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + validate_warmups_listener.ProgressListener 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.SynchronizerValidator 0.0% + (untracked) 0.0% + validate_warmups_listener.DataInitialization 0.0% + Input data preparation 6.5% ### + gpu_input_reset 3.8% # + gpu_input_reset_batched_inputs 2.4% # + gpu_input_reset_copy_inputs_d2d 1.2% + gpu_input_reset_fast_reset_output 0.2% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_copy_valid_to_gpu 0.0% + gpu_input_reset_slow_path 0.0% + gpu_input_reset_copy_swizzled 0.0% + gpu_input_reset_rotating_init 0.0% + gpu_input_reset_constant_inputs 0.0% + gpu_input_reset_init_cpu_inputs 0.0% + (untracked) 0.0% + cpu_data_init 1.8% + gpu_input_preparation 0.8% + gpu_input_prep_copy_valid_to_gpu 0.6% + gpu_input_prep_copy_swizzled 0.1% + gpu_input_prep_init_cpu_inputs 0.1% + gpu_input_prep_batched_inputs 0.0% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_fast_reset_output 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_rotating_init 0.0% + (untracked) 0.0% + rotating_buffer_preparation 0.0% + pre_solution_listener.DataInitialization 0.0% + pre_solution_mx_check 0.0% + (untracked) 0.0% + Reference GEMM (CPU) 4.6% ## + cpu_reference_gemm 4.6% ## + solve_cpu_fast 4.5% ## + solve_cpu_slow 0.1% + (untracked) 0.0% + pre_solution_listener.ReferenceValidator 0.0% + pre_solution_ref_skipped 0.0% + (untracked) 0.0% + Per-solution bookkeeping & reporting 1.1% + run_current_solution_check 0.2% + post_solution_reporting 0.2% + pre_solution_listener.BenchmarkTimer 0.2% + post_solution_log 0.1% + pre_solution_listener.AllSolutionsIterator 0.1% + pre_solution_listener.MetaResultReporter 0.1% + solution_selection 0.1% + pre_solution_listener.ProgressListener 0.1% + post_solution_validation 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + run_iteration_teardown 0.0% + post_problem 0.0% + solution_teardown 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + post_solution_perf_calc 0.0% + post_solution_sol_advance 0.0% + post_solution_perf_reset 0.0% + problem_report_index 0.0% + finalize_report 0.0% + problem_teardown 0.0% + post_solution_result_file 0.0% + Library logic (Python) 1.0% + python_logic_write 0.9% + python_logic_parse_solutions 0.1% + python_logic_analyze 0.0% + Library client (Python) 0.7% + python_library_client_create_library 0.5% + python_library_client_execution 0.2% + python_library_client_subprocess 0.2% + python_library_client_write_run_script 0.0% + (untracked) 0.0% + python_library_client_write_configs 0.0% + python_library_client_setup 0.0% + MEASUREMENT OVERHEAD (instrumentation) 13.0% ###### + flush_timing_buffer 10.8% ##### + main_timing_calibration 1.3% + timing_overhead 0.4% + (timer overhead subtracted from parent timers) 0.4% + SHARED: cache & solution files 0.1% + python_cache_check 0.1% + python_write_solutions 0.0% + python_wsol_prepare 0.0% + python_wsol_prepare_cache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + UNATTRIBUTED (untracked time & timer overhead) 0.1% + (untracked in python_benchmark_problems) 0.4% + (untracked in validate_warmups) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in python_library_logic) 0.0% + (untracked in python_client_writer) 0.0% + (children exceed benchmark_runs) -0.0% + (children exceed validate_enqueues_listener.BenchmarkTimer) -0.0% + (children exceed post_solution) -0.0% + (children exceed python_client_execution) -0.4% + +KEY INSIGHTS (% of wall clock) +------------------------------------------------------------------------------------------------------------------------------------ + Client launch (startup, setup & teardown) 468.44s (56.5%) + MEASUREMENT OVERHEAD (instrumentation) 104.52s (12.6%) + Kernel launch & execution 67.79s (8.2%) + Validation 66.75s (8.1%) + Input data preparation 53.73s (6.5%) + Reference GEMM (CPU) 38.35s (4.6%) + Per-solution bookkeeping & reporting 9.26s (1.1%) + Library logic (Python) 8.38s (1.0%) + Library client (Python) 6.21s (0.7%) + SHARED: cache & solution files 0.97s (0.1%) + UNATTRIBUTED (untracked time & timer overhead) 0.80s (0.1%) + + CPU reference / GPU execution ratio: 125.4x + >>> CPU reference > 10x GPU execution time! + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +------------------------------------------------------------------------------------------------------------------------------------ + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + ---------------------------------------------------------------------------------------------------------------------------------- + 2160 10240 16384 1 BFloat16 BFloat16 5439.82 0.00 + 10240 2160 16384 1 BFloat16 BFloat16 5283.25 0.00 + 4096 4096 4096 1 Half Half 1025.10 0.00 + 4111 4111 4111 1 Float8_fnuz Float8_fnuz 942.27 0.00 + 4111 4111 4111 1 Float8_fnuz Float8_fnuz 874.63 0.00 + 4111 4111 4111 1 Float8_fnuz Float 771.01 0.00 + 4111 4111 4111 1 Float8_fnuz Half 682.97 0.00 + 4111 4111 4111 1 Float8_fnuz Float 680.12 0.00 + 4111 4111 4111 1 BFloat8_fnuz BFloat16 623.44 0.00 + 4111 4111 4111 1 Float8_fnuz Float 608.52 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_build/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_build/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..dbeb98c4f05d --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_build/AGGREGATE_analysis.txt @@ -0,0 +1,167 @@ +======================================================================================================================== +TIMING ANALYSIS SUMMARY +======================================================================================================================== + +TIMING BY PURPOSE +------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 2,134,428.47 100.0% + BUILD: solution generation & compile 2,086,540.33 97.8% 97.8% + Solution generation & build (Python) 2,086,540.33 100.0% 97.8% + python_kernel_compilation 441 1,666,977.28 3,779.99 79.9% 78.1% + python_kernel_codegen 441 455,097.09 1,031.97 27.3% 21.3% + python_kernel_write_assemble 441 158,059.56 358.41 9.5% 7.4% + python_kernel_bench_postprocess 441 123,716.13 280.54 7.4% 5.8% + python_benchpost_lib_construction 441 82,981.66 188.17 67.1% 3.9% + python_benchpost_library_write 441 28,464.85 64.55 23.0% 1.3% + python_benchpost_naming 441 12,141.42 27.53 9.8% 0.6% + python_benchpost_client_config 441 103.14 0.23 0.1% 0.0% + (untracked) 25.05 0.0% 0.0% + python_kernel_bench_setup 441 123,611.11 280.30 7.4% 5.8% + python_kernel_validate 441 73,242.57 166.08 4.4% 3.4% + python_kernel_setup 441 31,789.44 72.08 1.9% 1.5% + python_kernel_build_co 441 20,337.45 46.12 1.2% 1.0% + python_kernel_write_helpers 441 8,028.18 18.20 0.5% 0.4% + (untracked) 673,095.75 40.4% 31.5% + python_solution_generation 441 419,563.05 951.39 20.1% 19.7% + python_solgen_forked_solutions 441 418,462.19 948.89 99.7% 19.6% + python_solgen_custom_kernels 441 1,088.92 2.47 0.3% 0.1% + python_solgen_fork_permutations 441 3.29 0.01 0.0% 0.0% + (untracked) 8.65 0.0% 0.0% + SHARED: cache & solution files 33,006.70 1.5% 1.5% + python_write_solutions 441 20,805.35 47.18 63.0% 1.0% + python_wsol_dump 441 20,147.83 45.69 96.8% 0.9% + python_wsol_prepare 441 544.88 1.24 2.6% 0.0% + python_wsol_prepare_nocache 441 543.71 1.23 99.8% 0.0% + (untracked) 1.18 0.2% 0.0% + python_wsol_header 441 8.86 0.02 0.0% 0.0% + (untracked) 103.78 0.5% 0.0% + python_solution_indexing 441 11,653.85 26.43 35.3% 0.5% + python_write_cache 441 546.90 1.24 1.7% 0.0% + python_cache_check 441 0.61 0.00 0.0% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 14,881.43 0.7% 0.7% + (untracked in python_benchmark_problems) 14,881.43 100.0% 0.7% + +TIMING BY TIMER NESTING (legacy view) +------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------ + python_benchmark_problems 123 2,134,428.47 17,353.08 100.0% + -------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 441 1,666,977.28 3,779.99 78.1% 78.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 673,095.75 40.4% 31.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 441 455,097.09 1,031.97 27.3% 21.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 441 158,059.56 358.41 9.5% 7.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 441 123,716.13 280.54 7.4% 5.8% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 441 82,981.66 188.17 67.1% 3.9% + python_benchpost_library_write 441 28,464.85 64.55 23.0% 1.3% + python_benchpost_naming 441 12,141.42 27.53 9.8% 0.6% + python_benchpost_client_config 441 103.14 0.23 0.1% 0.0% + (untracked) 25.05 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 441 123,611.11 280.30 7.4% 5.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 441 73,242.57 166.08 4.4% 3.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 441 31,789.44 72.08 1.9% 1.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 441 20,337.45 46.12 1.2% 1.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 441 8,028.18 18.20 0.5% 0.4% + -------------------------------------------------------------------------------------------------------------------- + python_solution_generation 441 419,563.05 951.39 19.7% 19.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_forked_solutions 441 418,462.19 948.89 99.7% 19.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_custom_kernels 441 1,088.92 2.47 0.3% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 8.65 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_fork_permutations 441 3.29 0.01 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------- + python_write_solutions 441 20,805.35 47.18 1.0% 1.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 441 20,147.83 45.69 96.8% 0.9% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 441 544.88 1.24 2.6% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_nocache 441 543.71 1.23 99.8% 0.0% + (untracked) 1.18 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 103.78 0.5% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 441 8.86 0.02 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------- + (untracked) 14,881.43 0.7% 0.7% + -------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 441 11,653.85 26.43 0.5% 0.5% + -------------------------------------------------------------------------------------------------------------------- + python_write_cache 441 546.90 1.24 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------- + python_cache_check 441 0.61 0.00 0.0% 0.0% + + Overlapping Totals (wrap or overlap other timers; not summed into the tree) + ---------------------------------------------------------------------------------------------------------------------- + python_kernel_build_src_co 441 667,895.15 1,514.50 31.3% + python_kernel_build_src_co.setup 441 23.62 0.05 0.0% + python_kernel_build_src_co.cache_check 441 1,087.91 2.47 0.1% + python_kernel_build_src_co.compile 97 665,369.87 6,859.48 31.2% + python_kernel_build_src_co.unbundle 97 884.38 9.12 0.0% + python_kernel_build_src_co.move 97 8.86 0.09 0.0% + python_kernel_build_src_co.cache_populate 97 113.60 1.17 0.0% + +------------------------------------------------------------------------------------------------------------------------ +HIERARCHY VALIDATION: OK (all child phases fit within their parents) + +TIME BREAKDOWN (visual, % of wall clock) +------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 100.0% ################################################## + BUILD: solution generation & compile 97.8% ################################################ + Solution generation & build (Python) 97.8% ################################################ + python_kernel_compilation 78.1% ####################################### + python_kernel_codegen 21.3% ########## + python_kernel_write_assemble 7.4% ### + python_kernel_bench_postprocess 5.8% ## + python_benchpost_lib_construction 3.9% # + python_benchpost_library_write 1.3% + python_benchpost_naming 0.6% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_bench_setup 5.8% ## + python_kernel_validate 3.4% # + python_kernel_setup 1.5% + python_kernel_build_co 1.0% + python_kernel_write_helpers 0.4% + (untracked) 31.5% ############### + python_solution_generation 19.7% ######### + python_solgen_forked_solutions 19.6% ######### + python_solgen_custom_kernels 0.1% + python_solgen_fork_permutations 0.0% + (untracked) 0.0% + SHARED: cache & solution files 1.5% + python_write_solutions 1.0% + python_wsol_dump 0.9% + python_wsol_prepare 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + python_solution_indexing 0.5% + python_write_cache 0.0% + python_cache_check 0.0% + UNATTRIBUTED (untracked time & timer overhead) 0.7% + (untracked in python_benchmark_problems) 0.7% + +KEY INSIGHTS (% of wall clock) +------------------------------------------------------------------------------------------------------------------------ + Solution generation & build (Python) 2086.54s (97.8%) + SHARED: cache & solution files 33.01s (1.5%) + UNATTRIBUTED (untracked time & timer overhead) 14.88s (0.7%) + + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_run/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_run/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..2beba206ba45 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx942/pre_run/AGGREGATE_analysis.txt @@ -0,0 +1,581 @@ +========================================================================================================================================== +TIMING ANALYSIS SUMMARY +========================================================================================================================================== + +TIMING BY PURPOSE +------------------------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 1,644,424.74 100.0% + RUN: benchmark client 1,534,553.50 93.3% 93.3% + Reference GEMM (CPU) 657,031.21 42.8% 40.0% + cpu_reference_gemm 4,029 656,952.69 163.06 100.0% 40.0% + solve_cpu_slow 1,462 624,526.75 427.17 95.1% 38.0% + solve_cpu_fast 2,626 32,418.16 12.35 4.9% 2.0% + (untracked) 7.78 0.0% 0.0% + pre_solution_listener.ReferenceValidator 1,286,411 78.52 0.00 0.0% 0.0% + pre_solution_ref_skipped 1,286,230 31.60 0.00 40.2% 0.0% + (untracked) 46.92 59.8% 0.0% + Client launch (startup, setup & teardown) 472,555.44 30.8% 28.7% + listener_setup 441 178,629.05 405.05 37.8% 10.9% + hip_initialization 441 167,171.50 379.07 35.4% 10.2% + process_exit 441 53,972.70 122.39 11.4% 3.3% + data_init_setup 441 18,751.47 42.52 4.0% 1.1% + library_loading 441 17,364.72 39.38 3.7% 1.1% + code_object_loading 441 14,211.38 32.23 3.0% 0.9% + main_teardown_locals 441 13,070.82 29.64 2.8% 0.8% + process_startup 441 8,165.24 18.52 1.7% 0.5% + lazy_loading_init 441 791.75 1.80 0.2% 0.0% + main_parse_args 441 238.62 0.54 0.1% 0.0% + main_problem_factory 441 90.54 0.21 0.0% 0.0% + (untracked) 90.54 100.0% 0.0% + reporter_setup 441 48.88 0.11 0.0% 0.0% + python_client_write_run_script 441 40.25 0.09 0.0% 0.0% + solution_iterator_setup 441 8.53 0.02 0.0% 0.0% + Validation 180,751.69 11.8% 11.0% + validate_warmups_listener.ReferenceValidator 1,229,030 96,941.00 0.08 53.6% 5.9% + validate_gpu_readback 1,042,067 59,865.63 0.06 61.8% 3.6% + validate_element_comparison 1,042,067 32,234.41 0.03 33.3% 2.0% + validate_result_buffer_alloc 1,042,067 3,897.37 0.00 4.0% 0.2% + validate_inputs_copy 1,036,487 175.42 0.00 0.2% 0.0% + validate_setup 1,037,703 40.51 0.00 0.0% 0.0% + validate_report 1,042,067 36.03 0.00 0.0% 0.0% + (untracked) 691.64 0.7% 0.0% + validate_warmups_listener.SynchronizerValidator 1,229,030 83,620.41 0.07 46.3% 5.1% + validate_synchronizer_check 1,229,052 83,620.41 0.07 100.0% 5.1% + validate_synchronizer_scan 253,775 55,951.86 0.22 66.9% 3.4% + validate_synchronizer_readback 253,775 27,406.94 0.11 32.8% 1.7% + validate_synchronizer_inactive 975,345 23.90 0.00 0.0% 0.0% + (untracked) 237.72 0.3% 0.0% + validate_warmups_listener.MetaResultReporter 1,229,030 52.86 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1,229,030 36.30 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 1,229,030 36.14 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1,229,030 32.80 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1,229,030 32.17 0.00 0.0% 0.0% + Input data preparation 128,321.13 8.4% 7.8% + gpu_input_reset 1,229,034 93,246.57 0.08 72.7% 5.7% + gpu_input_reset_copy_valid_to_gpu 137,906 39,099.41 0.28 41.9% 2.4% + gpu_input_reset_batched_inputs 658,043 36,356.02 0.06 39.0% 2.2% + gpu_input_reset_copy_inputs_d2d 658,043 11,435.13 0.02 12.3% 0.7% + gpu_input_reset_copy_swizzled 13,132 3,904.59 0.30 4.2% 0.2% + gpu_input_reset_fast_reset_output 570,863 2,009.75 0.00 2.2% 0.1% + gpu_input_reset_convert_to_inputs 658,043 105.63 0.00 0.1% 0.0% + gpu_input_reset_slow_path 658,043 16.27 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.04 0.02 0.0% 0.0% + gpu_input_reset_constant_inputs 5 0.03 0.01 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 5 0.01 0.00 0.0% 0.0% + (untracked) 319.71 0.3% 0.0% + cpu_data_init 4,029 22,931.94 5.69 17.9% 1.4% + gpu_input_preparation 4,035 11,636.68 2.88 9.1% 0.7% + gpu_input_prep_copy_valid_to_gpu 1,797 8,578.75 4.77 73.7% 0.5% + gpu_input_prep_init_cpu_inputs 3 1,718.87 572.96 14.8% 0.1% + gpu_input_prep_copy_swizzled 98 1,008.54 10.29 8.7% 0.1% + gpu_input_prep_batched_inputs 2,903 223.07 0.08 1.9% 0.0% + gpu_input_prep_copy_inputs_d2d 2,903 81.88 0.03 0.7% 0.0% + gpu_input_prep_fast_reset_output 1,119 12.93 0.01 0.1% 0.0% + gpu_input_prep_convert_to_inputs 2,903 2.05 0.00 0.0% 0.0% + gpu_input_prep_slow_path 2,903 0.13 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 4 0.05 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.02 0.02 0.0% 0.0% + (untracked) 10.40 0.1% 0.0% + rotating_buffer_preparation 4,035 438.30 0.11 0.3% 0.0% + pre_solution_listener.DataInitialization 1,286,411 67.63 0.00 0.1% 0.0% + pre_solution_mx_check 1,286,411 40.76 0.00 60.3% 0.0% + (untracked) 26.87 39.7% 0.0% + Per-solution bookkeeping & reporting 44,258.67 2.9% 2.7% + post_solution_log 1,286,411 15,785.47 0.01 35.7% 1.0% + pre_solution_listener.BenchmarkTimer 1,286,411 8,312.12 0.01 18.8% 0.5% + run_current_solution_check 1,286,411 5,748.15 0.00 13.0% 0.3% + post_solution_reporting 1,286,411 3,557.37 0.00 8.0% 0.2% + pre_solution_listener.ProgressListener 1,286,411 2,442.82 0.00 5.5% 0.1% + pre_solution_listener.AllSolutionsIterator 1,286,411 1,995.09 0.00 4.5% 0.1% + post_problem 4,035 1,691.10 0.42 3.8% 0.1% + run_iteration_teardown 1,229,034 1,365.25 0.00 3.1% 0.1% + solution_selection 1,286,411 790.82 0.00 1.8% 0.0% + pre_solution_listener.MetaResultReporter 1,286,411 716.53 0.00 1.6% 0.0% + post_solution_perf_calc 1,286,411 594.29 0.00 1.3% 0.0% + post_solution_validation 1,286,411 440.22 0.00 1.0% 0.0% + finalize_report 441 375.26 0.85 0.8% 0.0% + pre_solution_listener.SynchronizerValidator 1,286,411 225.16 0.00 0.5% 0.0% + post_solution_result_file 1,286,411 50.75 0.00 0.1% 0.0% + solution_teardown 1,286,411 50.70 0.00 0.1% 0.0% + post_solution_sol_advance 1,286,411 38.20 0.00 0.1% 0.0% + pre_solution_listener.HardwareMonitorListener 1,286,411 38.08 0.00 0.1% 0.0% + post_solution_perf_reset 1,286,411 32.98 0.00 0.1% 0.0% + problem_report_index 4,035 8.11 0.00 0.0% 0.0% + problem_teardown 4,035 0.20 0.00 0.0% 0.0% + Kernel launch & execution 35,439.17 2.3% 2.2% + warmup_runs 2,458,060 20,311.11 0.01 57.3% 1.2% + kernel_solving 1,229,034 6,192.62 0.01 17.5% 0.4% + validate_warmups_listener.BenchmarkTimer 1,229,030 5,863.27 0.00 16.5% 0.4% + validate_gpu_sync 1,229,030 5,845.64 0.00 99.7% 0.4% + (untracked) 17.64 0.3% 0.0% + benchmark_runs 1,229,034 1,673.68 0.00 4.7% 0.1% + benchmark_validate_enqueues 605 1,076.67 1.78 64.3% 0.1% + validate_enqueues_listener.BenchmarkTimer 605 600.07 0.99 55.7% 0.0% + benchmark_sleep 604 564.95 0.94 94.1% 0.0% + benchmark_event_sync 219 34.58 0.16 5.8% 0.0% + benchmark_event_elapsed_time 219 0.32 0.00 0.1% 0.0% + benchmark_event_destroy 219 0.22 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 605 211.26 0.35 19.6% 0.0% + validate_enqueues_listener.HardwareMonitorListener 605 171.94 0.28 16.0% 0.0% + validate_enqueues_listener.SynchronizerValidator 605 85.09 0.14 7.9% 0.0% + validate_enqueues_listener.ProgressListener 605 7.37 0.01 0.7% 0.0% + validate_enqueues_listener.MetaResultReporter 605 0.04 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 605 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 605 0.02 0.00 0.0% 0.0% + (untracked) 0.87 0.1% 0.0% + gpu_kernel_execution 605 440.16 0.73 26.3% 0.0% + benchmark_post_syncs 1,229,034 92.03 0.00 5.5% 0.0% + benchmark_pre_syncs 1,229,034 63.70 0.00 3.8% 0.0% + benchmark_events_setup 605 0.78 0.00 0.0% 0.0% + benchmark_events_teardown 605 0.36 0.00 0.0% 0.0% + warmup_events_setup 1,229,034 1,398.48 0.00 3.9% 0.1% + Library logic (Python) 9,347.35 0.6% 0.6% + python_logic_write 8 8,282.05 1,035.26 88.6% 0.5% + python_logic_parse_solutions 8 938.09 117.26 10.0% 0.1% + python_logic_analyze 8 127.21 15.90 1.4% 0.0% + Library client (Python) 6,848.84 0.4% 0.4% + python_library_client_create_library 1 4,719.06 4,719.06 68.9% 0.3% + python_library_client_execution 1 2,104.68 2,104.68 30.7% 0.1% + python_library_client_subprocess 1 2,103.74 2,103.74 100.0% 0.1% + python_library_client_write_run_script 1 0.16 0.16 0.0% 0.0% + (untracked) 0.78 0.0% 0.0% + python_library_client_write_configs 1 23.56 23.56 0.3% 0.0% + python_library_client_setup 1 1.54 1.54 0.0% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 108,054.22 6.6% 6.6% + flush_timing_buffer 443 89,840.08 202.80 83.1% 5.5% + main_timing_calibration 441 11,030.32 25.01 10.2% 0.7% + timing_overhead 441 3,716.45 8.43 3.4% 0.2% + (timer overhead subtracted from parent timers) 3,467.37 3.2% 0.2% + SHARED: cache & solution files 924.24 0.1% 0.1% + python_cache_check 441 836.07 1.90 90.5% 0.1% + python_write_solutions 441 88.17 0.20 9.5% 0.0% + python_wsol_prepare 441 63.41 0.14 71.9% 0.0% + python_wsol_prepare_cache 441 61.49 0.14 97.0% 0.0% + (untracked) 1.93 3.0% 0.0% + python_wsol_header 441 19.71 0.04 22.4% 0.0% + (untracked) 5.05 5.7% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 892.79 0.1% 0.1% + (untracked in python_benchmark_problems) 2,908.43 325.8% 0.2% + (untracked in validate_warmups) 564.34 63.2% 0.0% + (untracked in pre_solution) 512.63 57.4% 0.0% + (untracked in pre_problem) 67.88 7.6% 0.0% + (untracked in python_library_logic) 6.33 0.7% 0.0% + (untracked in python_client_writer) 1.08 0.1% 0.0% + (children exceed benchmark_runs) -12.92 -1.4% -0.0% + (children exceed post_solution) -26.81 -3.0% -0.0% + (children exceed validate_enqueues_listener.BenchmarkTimer) -34.53 -3.9% -0.0% + (children exceed validate_warmups_listener.SynchronizerValidator) -55.75 -6.2% -0.0% + (children exceed python_client_execution) -3,037.90 -340.3% -0.2% + +TIMING BY TIMER NESTING (legacy view) +------------------------------------------------------------------------------------------------------------------------------------------ + Category Count Total (ms) Mean (ms) % of Parent % of Wall +------------------------------------------------------------------------------------------------------------------------------------------ + python_benchmark_problems 123 1,624,753.78 13,209.38 98.8% + -------------------------------------------------------------------------------------------------------------------------------------- + python_client_execution 441 1,620,921.10 3,675.56 99.8% 98.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 785,274.07 48.4% 47.8% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_problem 4,035 679,952.51 168.51 86.6% 41.3% + cpu_reference_gemm 4,029 656,952.69 163.06 96.6% 40.0% + solve_cpu_slow 1,462 624,526.75 427.17 95.1% 38.0% + solve_cpu_fast 2,626 32,418.16 12.35 4.9% 2.0% + (untracked) 7.78 0.0% 0.0% + cpu_data_init 4,029 22,931.94 5.69 3.4% 1.4% + (untracked) 67.88 0.0% 0.0% + gpu_input_reset 1,229,034 93,246.57 0.08 11.9% 5.7% + gpu_input_reset_copy_valid_to_gpu 137,906 39,099.41 0.28 41.9% 2.4% + gpu_input_reset_batched_inputs 658,043 36,356.02 0.06 39.0% 2.2% + gpu_input_reset_copy_inputs_d2d 658,043 11,435.13 0.02 12.3% 0.7% + gpu_input_reset_copy_swizzled 13,132 3,904.59 0.30 4.2% 0.2% + gpu_input_reset_fast_reset_output 570,863 2,009.75 0.00 2.2% 0.1% + (untracked) 319.71 0.3% 0.0% + gpu_input_reset_convert_to_inputs 658,043 105.63 0.00 0.1% 0.0% + gpu_input_reset_slow_path 658,043 16.27 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.04 0.02 0.0% 0.0% + gpu_input_reset_constant_inputs 5 0.03 0.01 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 5 0.01 0.00 0.0% 0.0% + gpu_input_preparation 4,035 11,636.68 2.88 1.5% 0.7% + gpu_input_prep_copy_valid_to_gpu 1,797 8,578.75 4.77 73.7% 0.5% + gpu_input_prep_init_cpu_inputs 3 1,718.87 572.96 14.8% 0.1% + gpu_input_prep_copy_swizzled 98 1,008.54 10.29 8.7% 0.1% + gpu_input_prep_batched_inputs 2,903 223.07 0.08 1.9% 0.0% + gpu_input_prep_copy_inputs_d2d 2,903 81.88 0.03 0.7% 0.0% + gpu_input_prep_fast_reset_output 1,119 12.93 0.01 0.1% 0.0% + (untracked) 10.40 0.1% 0.0% + gpu_input_prep_convert_to_inputs 2,903 2.05 0.00 0.0% 0.0% + gpu_input_prep_slow_path 2,903 0.13 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 4 0.05 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.02 0.02 0.0% 0.0% + rotating_buffer_preparation 4,035 438.30 0.11 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 483,585.76 29.8% 29.4% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + listener_setup 441 178,629.05 405.05 36.9% 10.9% + hip_initialization 441 167,171.50 379.07 34.6% 10.2% + process_exit 441 53,972.70 122.39 11.2% 3.3% + data_init_setup 441 18,751.47 42.52 3.9% 1.1% + library_loading 441 17,364.72 39.38 3.6% 1.1% + code_object_loading 441 14,211.38 32.23 2.9% 0.9% + main_teardown_locals 441 13,070.82 29.64 2.7% 0.8% + main_timing_calibration 441 11,030.32 25.01 2.3% 0.7% + process_startup 441 8,165.24 18.52 1.7% 0.5% + lazy_loading_init 441 791.75 1.80 0.2% 0.0% + main_parse_args 441 238.62 0.54 0.0% 0.0% + main_problem_factory 441 90.54 0.21 0.0% 0.0% + (untracked) 90.54 100.0% 0.0% + reporter_setup 441 48.88 0.11 0.0% 0.0% + python_client_write_run_script 441 40.25 0.09 0.0% 0.0% + solution_iterator_setup 441 8.53 0.02 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 187,123.55 11.5% 11.4% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 1,229,030 187,123.55 0.15 100.0% 11.4% + validate_warmups_listener.ReferenceValidator 1,229,030 96,941.00 0.08 51.8% 5.9% + validate_gpu_readback 1,042,067 59,865.63 0.06 61.8% 3.6% + validate_element_comparison 1,042,067 32,234.41 0.03 33.3% 2.0% + validate_result_buffer_alloc 1,042,067 3,897.37 0.00 4.0% 0.2% + (untracked) 691.64 0.7% 0.0% + validate_inputs_copy 1,036,487 175.42 0.00 0.2% 0.0% + validate_setup 1,037,703 40.51 0.00 0.0% 0.0% + validate_report 1,042,067 36.03 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1,229,030 83,564.67 0.07 44.7% 5.1% + validate_synchronizer_check 1,229,052 83,620.41 0.07 100.1% 5.1% + validate_synchronizer_scan 253,775 55,951.86 0.22 66.9% 3.4% + validate_synchronizer_readback 253,775 27,406.94 0.11 32.8% 1.7% + (untracked) 237.72 0.3% 0.0% + validate_synchronizer_inactive 975,345 23.90 0.00 0.0% 0.0% + validate_warmups_listener.BenchmarkTimer 1,229,030 5,863.27 0.00 3.1% 0.4% + validate_gpu_sync 1,229,030 5,845.64 0.00 99.7% 0.4% + (untracked) 17.64 0.3% 0.0% + (untracked) 564.34 0.3% 0.0% + validate_warmups_listener.MetaResultReporter 1,229,030 52.86 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1,229,030 36.30 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 1,229,030 36.14 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1,229,030 32.80 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1,229,030 32.17 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 93,556.53 5.8% 5.7% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 443 89,840.08 202.80 96.0% 5.5% + timing_overhead 441 3,716.45 8.43 4.0% 0.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 74,419.09 4.6% 4.5% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + post_solution 1,286,411 20,472.48 0.02 27.5% 1.2% + post_solution_log 1,286,411 15,785.47 0.01 77.1% 1.0% + post_solution_reporting 1,286,411 3,557.37 0.00 17.4% 0.2% + post_solution_perf_calc 1,286,411 594.29 0.00 2.9% 0.0% + post_solution_validation 1,286,411 440.22 0.00 2.2% 0.0% + post_solution_result_file 1,286,411 50.75 0.00 0.2% 0.0% + post_solution_sol_advance 1,286,411 38.20 0.00 0.2% 0.0% + post_solution_perf_reset 1,286,411 32.98 0.00 0.2% 0.0% + warmup_runs 2,458,060 20,311.11 0.01 27.3% 1.2% + pre_solution 1,286,411 14,388.58 0.01 19.3% 0.9% + pre_solution_listener.BenchmarkTimer 1,286,411 8,312.12 0.01 57.8% 0.5% + pre_solution_listener.ProgressListener 1,286,411 2,442.82 0.00 17.0% 0.1% + pre_solution_listener.AllSolutionsIterator 1,286,411 1,995.09 0.00 13.9% 0.1% + pre_solution_listener.MetaResultReporter 1,286,411 716.53 0.00 5.0% 0.0% + (untracked) 512.63 3.6% 0.0% + pre_solution_listener.SynchronizerValidator 1,286,411 225.16 0.00 1.6% 0.0% + pre_solution_listener.ReferenceValidator 1,286,411 78.52 0.00 0.5% 0.0% + (untracked) 46.92 59.8% 0.0% + pre_solution_ref_skipped 1,286,230 31.60 0.00 40.2% 0.0% + pre_solution_listener.DataInitialization 1,286,411 67.63 0.00 0.5% 0.0% + pre_solution_mx_check 1,286,411 40.76 0.00 60.3% 0.0% + (untracked) 26.87 39.7% 0.0% + pre_solution_listener.HardwareMonitorListener 1,286,411 38.08 0.00 0.3% 0.0% + kernel_solving 1,229,034 6,192.62 0.01 8.3% 0.4% + run_current_solution_check 1,286,411 5,748.15 0.00 7.7% 0.3% + post_problem 4,035 1,691.10 0.42 2.3% 0.1% + benchmark_runs 1,229,034 1,626.23 0.00 2.2% 0.1% + benchmark_validate_enqueues 605 1,042.13 1.72 64.1% 0.1% + validate_enqueues_listener.BenchmarkTimer 605 565.54 0.93 54.3% 0.0% + benchmark_sleep 604 564.95 0.94 99.9% 0.0% + benchmark_event_sync 219 34.58 0.16 6.1% 0.0% + benchmark_event_elapsed_time 219 0.32 0.00 0.1% 0.0% + benchmark_event_destroy 219 0.22 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 605 211.26 0.35 20.3% 0.0% + validate_enqueues_listener.HardwareMonitorListener 605 171.94 0.28 16.5% 0.0% + validate_enqueues_listener.SynchronizerValidator 605 85.09 0.14 8.2% 0.0% + validate_enqueues_listener.ProgressListener 605 7.37 0.01 0.7% 0.0% + (untracked) 0.87 0.1% 0.0% + validate_enqueues_listener.MetaResultReporter 605 0.04 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 605 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 605 0.02 0.00 0.0% 0.0% + gpu_kernel_execution 605 440.16 0.73 27.1% 0.0% + benchmark_post_syncs 1,229,034 92.03 0.00 5.7% 0.0% + benchmark_pre_syncs 1,229,034 63.70 0.00 3.9% 0.0% + benchmark_events_setup 605 0.78 0.00 0.0% 0.0% + benchmark_events_teardown 605 0.36 0.00 0.0% 0.0% + warmup_events_setup 1,229,034 1,398.48 0.00 1.9% 0.1% + run_iteration_teardown 1,229,034 1,365.25 0.00 1.8% 0.1% + solution_selection 1,286,411 790.82 0.00 1.1% 0.0% + finalize_report 441 375.26 0.85 0.5% 0.0% + solution_teardown 1,286,411 50.70 0.00 0.1% 0.0% + problem_report_index 4,035 8.11 0.00 0.0% 0.0% + problem_teardown 4,035 0.20 0.00 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------------------------- + (untracked) 2,908.43 0.2% 0.2% + -------------------------------------------------------------------------------------------------------------------------------------- + python_cache_check 441 836.07 1.90 0.1% 0.1% + -------------------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 441 88.17 0.20 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 441 63.41 0.14 71.9% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_cache 441 61.49 0.14 97.0% 0.0% + (untracked) 1.93 3.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 441 19.71 0.04 22.4% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 5.05 5.7% 0.0% + ======================================================================================================================================== + python_library_logic 8 9,353.68 1,169.21 0.6% + -------------------------------------------------------------------------------------------------------------------------------------- + python_logic_write 8 8,282.05 1,035.26 88.5% 0.5% + -------------------------------------------------------------------------------------------------------------------------------------- + python_logic_parse_solutions 8 938.09 117.26 10.0% 0.1% + -------------------------------------------------------------------------------------------------------------------------------------- + python_logic_analyze 8 127.21 15.90 1.4% 0.0% + -------------------------------------------------------------------------------------------------------------------------------------- + (untracked) 6.33 0.1% 0.0% + ======================================================================================================================================== + python_client_writer 1 6,849.91 6,849.91 0.4% + -------------------------------------------------------------------------------------------------------------------------------------- + python_library_client_create_library 1 4,719.06 4,719.06 68.9% 0.3% + -------------------------------------------------------------------------------------------------------------------------------------- + python_library_client_execution 1 2,104.68 2,104.68 30.7% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_subprocess 1 2,103.74 2,103.74 100.0% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 0.78 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_write_run_script 1 0.16 0.16 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------------------------- + python_library_client_write_configs 1 23.56 23.56 0.3% 0.0% + -------------------------------------------------------------------------------------------------------------------------------------- + python_library_client_setup 1 1.54 1.54 0.0% 0.0% + -------------------------------------------------------------------------------------------------------------------------------------- + (untracked) 1.08 0.0% 0.0% + + Unknown Categories + ---------------------------------------------------------------------------------------------------------------------------------------- + icache_rotate_extra_copies_loading 1 5.72 5.72 0.0% + + Overlapping Totals (wrap or overlap other timers; not summed into the tree) + ---------------------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 441 1,624,103.51 3,682.77 98.8% + main_total 441 1,459,029.87 3,308.46 88.7% + problem_total 4,035 1,050,297.13 260.30 63.9% + solution_total 1,286,411 356,477.99 0.28 21.7% + run_iteration_total 1,229,034 313,383.40 0.25 19.1% + validate_total 1,037,703 96,753.31 0.09 5.9% + validate_check_results_typed 1,042,067 96,457.33 0.09 5.9% + +------------------------------------------------------------------------------------------------------------------------------------------ +HIERARCHY VALIDATION WARNINGS +------------------------------------------------------------------------------------------------------------------------------------------ + WARNING: Child phases of python_client_execution (1,626,543.75 ms) exceed parent (1,624,388.33 ms) by 2,155.42 ms (0.1%) + WARNING: Child phases of validate_enqueues_listener.BenchmarkTimer (600.07 ms) exceed parent (565.61 ms) by 34.47 ms (6.1%) + +TIME BREAKDOWN (visual, % of wall clock) +------------------------------------------------------------------------------------------------------------------------------------------ + TOTAL RUNTIME 100.0% ################################################## + RUN: benchmark client 93.3% ############################################## + Reference GEMM (CPU) 40.0% ################### + cpu_reference_gemm 40.0% ################### + solve_cpu_slow 38.0% ################## + solve_cpu_fast 2.0% + (untracked) 0.0% + pre_solution_listener.ReferenceValidator 0.0% + pre_solution_ref_skipped 0.0% + (untracked) 0.0% + Client launch (startup, setup & teardown) 28.7% ############## + listener_setup 10.9% ##### + hip_initialization 10.2% ##### + process_exit 3.3% # + data_init_setup 1.1% + library_loading 1.1% + code_object_loading 0.9% + main_teardown_locals 0.8% + process_startup 0.5% + lazy_loading_init 0.0% + main_parse_args 0.0% + main_problem_factory 0.0% + (untracked) 0.0% + reporter_setup 0.0% + python_client_write_run_script 0.0% + solution_iterator_setup 0.0% + Validation 11.0% ##### + validate_warmups_listener.ReferenceValidator 5.9% ## + validate_gpu_readback 3.6% # + validate_element_comparison 2.0% + validate_result_buffer_alloc 0.2% + validate_inputs_copy 0.0% + validate_setup 0.0% + validate_report 0.0% + (untracked) 0.0% + validate_warmups_listener.SynchronizerValidator 5.1% ## + validate_synchronizer_check 5.1% ## + validate_synchronizer_scan 3.4% # + validate_synchronizer_readback 1.7% + validate_synchronizer_inactive 0.0% + (untracked) 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + validate_warmups_listener.ProgressListener 0.0% + validate_warmups_listener.DataInitialization 0.0% + Input data preparation 7.8% ### + gpu_input_reset 5.7% ## + gpu_input_reset_copy_valid_to_gpu 2.4% # + gpu_input_reset_batched_inputs 2.2% # + gpu_input_reset_copy_inputs_d2d 0.7% + gpu_input_reset_copy_swizzled 0.2% + gpu_input_reset_fast_reset_output 0.1% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_slow_path 0.0% + gpu_input_reset_rotating_init 0.0% + gpu_input_reset_constant_inputs 0.0% + gpu_input_reset_init_cpu_inputs 0.0% + (untracked) 0.0% + cpu_data_init 1.4% + gpu_input_preparation 0.7% + gpu_input_prep_copy_valid_to_gpu 0.5% + gpu_input_prep_init_cpu_inputs 0.1% + gpu_input_prep_copy_swizzled 0.1% + gpu_input_prep_batched_inputs 0.0% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_fast_reset_output 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_rotating_init 0.0% + (untracked) 0.0% + rotating_buffer_preparation 0.0% + pre_solution_listener.DataInitialization 0.0% + pre_solution_mx_check 0.0% + (untracked) 0.0% + Per-solution bookkeeping & reporting 2.7% # + post_solution_log 1.0% + pre_solution_listener.BenchmarkTimer 0.5% + run_current_solution_check 0.3% + post_solution_reporting 0.2% + pre_solution_listener.ProgressListener 0.1% + pre_solution_listener.AllSolutionsIterator 0.1% + post_problem 0.1% + run_iteration_teardown 0.1% + solution_selection 0.0% + pre_solution_listener.MetaResultReporter 0.0% + post_solution_perf_calc 0.0% + post_solution_validation 0.0% + finalize_report 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + post_solution_result_file 0.0% + solution_teardown 0.0% + post_solution_sol_advance 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + post_solution_perf_reset 0.0% + problem_report_index 0.0% + problem_teardown 0.0% + Kernel launch & execution 2.2% # + warmup_runs 1.2% + kernel_solving 0.4% + validate_warmups_listener.BenchmarkTimer 0.4% + validate_gpu_sync 0.4% + (untracked) 0.0% + benchmark_runs 0.1% + benchmark_validate_enqueues 0.1% + validate_enqueues_listener.BenchmarkTimer 0.0% + benchmark_sleep 0.0% + benchmark_event_sync 0.0% + benchmark_event_elapsed_time 0.0% + benchmark_event_destroy 0.0% + validate_enqueues_listener.ReferenceValidator 0.0% + validate_enqueues_listener.HardwareMonitorListener 0.0% + validate_enqueues_listener.SynchronizerValidator 0.0% + validate_enqueues_listener.ProgressListener 0.0% + validate_enqueues_listener.MetaResultReporter 0.0% + validate_enqueues_listener.DataInitialization 0.0% + validate_enqueues_listener.AllSolutionsIterator 0.0% + (untracked) 0.0% + gpu_kernel_execution 0.0% + benchmark_post_syncs 0.0% + benchmark_pre_syncs 0.0% + benchmark_events_setup 0.0% + benchmark_events_teardown 0.0% + warmup_events_setup 0.1% + Library logic (Python) 0.6% + python_logic_write 0.5% + python_logic_parse_solutions 0.1% + python_logic_analyze 0.0% + Library client (Python) 0.4% + python_library_client_create_library 0.3% + python_library_client_execution 0.1% + python_library_client_subprocess 0.1% + python_library_client_write_run_script 0.0% + (untracked) 0.0% + python_library_client_write_configs 0.0% + python_library_client_setup 0.0% + MEASUREMENT OVERHEAD (instrumentation) 6.6% ### + flush_timing_buffer 5.5% ## + main_timing_calibration 0.7% + timing_overhead 0.2% + (timer overhead subtracted from parent timers) 0.2% + SHARED: cache & solution files 0.1% + python_cache_check 0.1% + python_write_solutions 0.0% + python_wsol_prepare 0.0% + python_wsol_prepare_cache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + UNATTRIBUTED (untracked time & timer overhead) 0.1% + (untracked in python_benchmark_problems) 0.2% + (untracked in validate_warmups) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in python_library_logic) 0.0% + (untracked in python_client_writer) 0.0% + (children exceed benchmark_runs) -0.0% + (children exceed post_solution) -0.0% + (children exceed validate_enqueues_listener.BenchmarkTimer) -0.0% + (children exceed validate_warmups_listener.SynchronizerValidator) -0.0% + (children exceed python_client_execution) -0.2% + +KEY INSIGHTS (% of wall clock) +------------------------------------------------------------------------------------------------------------------------------------------ + Reference GEMM (CPU) 657.03s (40.0%) + Client launch (startup, setup & teardown) 472.56s (28.7%) + Validation 180.75s (11.0%) + Input data preparation 128.32s (7.8%) + MEASUREMENT OVERHEAD (instrumentation) 104.59s (6.4%) + Per-solution bookkeeping & reporting 44.26s (2.7%) + Kernel launch & execution 35.44s (2.2%) + Library logic (Python) 9.35s (0.6%) + Library client (Python) 6.85s (0.4%) + SHARED: cache & solution files 0.92s (0.1%) + UNATTRIBUTED (untracked time & timer overhead) 0.89s (0.1%) + + CPU reference / GPU execution ratio: 1492.5x + >>> CPU reference > 10x GPU execution time! + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +------------------------------------------------------------------------------------------------------------------------------------------ + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + ---------------------------------------------------------------------------------------------------------------------------------------- + 4111 4111 4111 1 Float8_fnuz Float8_fnuz 233400.59 0.00 + 4111 4111 4111 1 Float8_fnuz Float8_fnuz 221845.69 0.00 + 1024 1024 1024 1 Float8_fnuz Float8_fnuz 7128.91 0.00 + 1024 1024 1024 1 Float8_fnuz Float8_fnuz 6008.58 0.00 + 2160 10240 16384 1 BFloat16 BFloat16 4751.19 0.00 + 10240 2160 16384 1 BFloat16 BFloat16 3400.16 0.00 + 4096 4096 4096 1 Half Half 1175.05 0.00 + 512 256 511 1 Float8_fnuz Float 903.25 0.00 + 4111 4111 4111 1 Float8_fnuz Half 884.17 0.00 + 511 256 1024 1 Half Half 840.64 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_build/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_build/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..96a7afcf836f --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_build/AGGREGATE_analysis.txt @@ -0,0 +1,575 @@ +============================================================================================================================= +TIMING ANALYSIS SUMMARY +============================================================================================================================= + +TIMING BY PURPOSE +----------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +----------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 1,286,569.24 100.0% + BUILD: solution generation & compile 1,241,290.68 96.5% 96.5% + Solution generation & build (Python) 1,219,840.35 98.3% 94.8% + python_kernel_compilation 1,862 904,610.66 485.83 74.2% 70.3% + python_kernel_codegen 214 301,368.91 1,408.27 33.3% 23.4% + python_kernel_write_assemble 856 236,194.18 275.93 26.1% 18.4% + python_kernel_build_src_co_wait 856 139,026.68 162.41 15.4% 10.8% + python_kernel_bench_postprocess 856 106,019.05 123.85 11.7% 8.2% + python_benchpost_lib_construction 856 69,091.03 80.71 65.2% 5.4% + python_benchpost_library_write 856 26,755.03 31.26 25.2% 2.1% + python_benchpost_naming 856 9,931.36 11.60 9.4% 0.8% + python_benchpost_client_config 856 205.40 0.24 0.2% 0.0% + (untracked) 36.24 0.0% 0.0% + python_kernel_validate 856 58,429.40 68.26 6.5% 4.5% + python_kernel_build_co 856 25,692.09 30.01 2.8% 2.0% + python_kernel_setup 856 21,607.69 25.24 2.4% 1.7% + python_kernel_write_helpers 856 7,968.35 9.31 0.9% 0.6% + python_kernel_bench_setup 856 4,549.36 5.31 0.5% 0.4% + (untracked) 3,754.95 0.4% 0.3% + python_solution_generation 824 315,229.69 382.56 25.8% 24.5% + python_solgen_forked_solutions 824 315,154.32 382.47 100.0% 24.5% + python_solgen_custom_kernels 824 57.74 0.07 0.0% 0.0% + python_solgen_fork_permutations 824 4.61 0.01 0.0% 0.0% + (untracked) 13.02 0.0% 0.0% + Ductile GA search (Python) 21,450.33 1.7% 1.7% + python_ductile_initial_sampling 8 15,265.87 1,908.23 71.2% 1.2% + python_ductile_generate_solutions 32 4,525.43 141.42 21.1% 0.4% + python_ductile_ga_between_evaluations 8 1,609.22 201.15 7.5% 0.1% + python_ductile_setup 8 35.96 4.50 0.2% 0.0% + python_ductile_clear_artifacts 24 7.81 0.33 0.0% 0.0% + python_ductile_read_scores 32 5.63 0.18 0.0% 0.0% + python_ductile_verification 8 0.40 0.05 0.0% 0.0% + SHARED: cache & solution files 23,439.24 1.8% 1.8% + python_write_solutions 856 14,118.81 16.49 60.2% 1.1% + python_wsol_dump 856 13,565.73 15.85 96.1% 1.1% + python_wsol_prepare 856 452.00 0.53 3.2% 0.0% + python_wsol_prepare_nocache 856 450.45 0.53 99.7% 0.0% + (untracked) 1.54 0.3% 0.0% + python_wsol_header 856 13.40 0.02 0.1% 0.0% + (untracked) 87.68 0.6% 0.0% + python_solution_indexing 856 8,401.41 9.81 35.8% 0.7% + python_write_cache 856 918.42 1.07 3.9% 0.1% + python_cache_check 832 0.61 0.00 0.0% 0.0% + RUN: benchmark client 9,953.37 0.8% 0.8% + Client launch (startup, setup & teardown) 8,872.09 89.1% 0.7% + hip_initialization 32 5,103.79 159.49 57.5% 0.4% + process_exit 32 2,186.50 68.33 24.6% 0.2% + main_problem_factory 32 640.52 20.02 7.2% 0.0% + factory_hip_get_device_properties 4 637.01 159.25 99.5% 0.0% + (untracked) 3.52 0.5% 0.0% + process_startup 32 480.60 15.02 5.4% 0.0% + main_teardown_locals 32 150.57 4.71 1.7% 0.0% + data_init_setup 32 97.89 3.06 1.1% 0.0% + code_object_loading 32 92.63 2.89 1.0% 0.0% + library_loading 32 62.38 1.95 0.7% 0.0% + lazy_loading_init 32 43.61 1.36 0.5% 0.0% + main_parse_args 32 10.34 0.32 0.1% 0.0% + python_client_write_run_script 32 1.41 0.04 0.0% 0.0% + reporter_setup 32 1.14 0.04 0.0% 0.0% + listener_setup 32 0.41 0.01 0.0% 0.0% + solution_iterator_setup 32 0.28 0.01 0.0% 0.0% + Input data preparation 1,058.18 10.6% 0.1% + gpu_input_preparation 40 943.11 23.58 89.1% 0.1% + gpu_input_prep_copy_valid_to_gpu 40 472.41 11.81 50.1% 0.0% + gpu_input_prep_init_cpu_inputs 39 404.45 10.37 42.9% 0.0% + gpu_input_prep_copy_swizzled 4 61.73 15.43 6.5% 0.0% + gpu_input_prep_batched_inputs 40 2.89 0.07 0.3% 0.0% + gpu_input_prep_copy_inputs_d2d 40 1.06 0.03 0.1% 0.0% + gpu_input_prep_constant_inputs 39 0.43 0.01 0.0% 0.0% + gpu_input_prep_convert_to_inputs 40 0.09 0.00 0.0% 0.0% + gpu_input_prep_slow_path 40 0.00 0.00 0.0% 0.0% + (untracked) 0.06 0.0% 0.0% + cpu_data_init 1 74.69 74.69 7.1% 0.0% + pre_solution_listener.DataInitialization 585 33.93 0.06 3.2% 0.0% + pre_solution_mx_init_data 130 19.99 0.15 58.9% 0.0% + pre_solution_mx_side_swizzle 8 19.95 2.49 99.8% 0.0% + pre_solution_mx_side_cache_hit 252 0.02 0.00 0.1% 0.0% + (untracked) 0.03 0.1% 0.0% + pre_solution_mx_cpu_sync_no_regen 130 10.19 0.08 30.0% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 130 3.67 0.03 10.8% 0.0% + pre_solution_mx_check 585 0.03 0.00 0.1% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 130 0.02 0.00 0.0% 0.0% + (untracked) 0.03 0.1% 0.0% + rotating_buffer_preparation 40 6.41 0.16 0.6% 0.0% + gpu_input_reset 1 0.03 0.03 0.0% 0.0% + gpu_input_reset_copy_inputs_d2d 1 0.03 0.03 95.3% 0.0% + gpu_input_reset_batched_inputs 1 0.00 0.00 1.7% 0.0% + gpu_input_reset_convert_to_inputs 1 0.00 0.00 1.4% 0.0% + gpu_input_reset_copy_swizzled 1 0.00 0.00 0.9% 0.0% + gpu_input_reset_copy_valid_to_gpu 1 0.00 0.00 0.6% 0.0% + gpu_input_reset_slow_path 1 0.00 0.00 0.1% 0.0% + Per-solution bookkeeping & reporting 12.66 0.1% 0.0% + pre_solution_listener.BenchmarkTimer 585 3.85 0.01 30.4% 0.0% + run_current_solution_check 585 1.66 0.00 13.1% 0.0% + post_problem 40 1.52 0.04 12.0% 0.0% + pre_solution_listener.ProgressListener 585 1.29 0.00 10.2% 0.0% + pre_solution_listener.AllSolutionsIterator 585 1.16 0.00 9.2% 0.0% + post_solution_reporting 585 1.12 0.00 8.8% 0.0% + post_solution_log 585 0.60 0.00 4.8% 0.0% + pre_solution_listener.MetaResultReporter 585 0.49 0.00 3.8% 0.0% + finalize_report 32 0.40 0.01 3.1% 0.0% + solution_selection 585 0.15 0.00 1.2% 0.0% + problem_report_index 40 0.13 0.00 1.1% 0.0% + pre_solution_listener.SynchronizerValidator 585 0.10 0.00 0.8% 0.0% + problem_teardown 40 0.08 0.00 0.6% 0.0% + solution_teardown 585 0.02 0.00 0.2% 0.0% + pre_solution_listener.HardwareMonitorListener 585 0.02 0.00 0.1% 0.0% + run_iteration_teardown 1 0.01 0.01 0.1% 0.0% + post_solution_result_file 585 0.01 0.00 0.1% 0.0% + post_solution_validation 585 0.01 0.00 0.1% 0.0% + post_solution_perf_calc 585 0.01 0.00 0.1% 0.0% + post_solution_sol_advance 585 0.01 0.00 0.1% 0.0% + post_solution_perf_reset 585 0.01 0.00 0.1% 0.0% + Validation 9.84 0.1% 0.0% + validate_warmups_listener.ReferenceValidator 1 9.22 9.22 93.7% 0.0% + validate_gpu_readback 1 7.91 7.91 85.9% 0.0% + validate_result_buffer_alloc 1 1.29 1.29 14.0% 0.0% + validate_element_comparison 1 0.01 0.01 0.1% 0.0% + validate_inputs_copy 1 0.00 0.00 0.0% 0.0% + validate_setup 1 0.00 0.00 0.0% 0.0% + validate_report 1 0.00 0.00 0.0% 0.0% + synchronizer_check 1 0.62 0.62 6.3% 0.0% + synchronizer_reduce 1 0.62 0.62 100.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.MetaResultReporter 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1 0.00 0.00 0.0% 0.0% + Kernel launch & execution 0.43 0.0% 0.0% + run_iteration_sync 1 0.18 0.18 41.6% 0.0% + warmup_runs 2 0.12 0.06 28.7% 0.0% + kernel_solving 1 0.06 0.06 15.0% 0.0% + warmup_first_sync 1 0.04 0.04 8.5% 0.0% + warmup_events_setup 1 0.02 0.02 4.8% 0.0% + validate_warmups_listener.BenchmarkTimer 1 0.01 0.01 1.3% 0.0% + validate_gpu_sync 1 0.01 0.01 100.0% 0.0% + benchmark_runs 1 0.00 0.00 0.1% 0.0% + benchmark_pre_syncs 1 0.00 0.00 51.4% 0.0% + benchmark_post_syncs 1 0.00 0.00 48.6% 0.0% + Reference GEMM (CPU) 0.17 0.0% 0.0% + cpu_reference_gemm 1 0.15 0.15 88.6% 0.0% + solve_cpu_slow 1 0.15 0.15 100.0% 0.0% + pre_solution_listener.ReferenceValidator 585 0.02 0.00 11.4% 0.0% + pre_solution_ref_skipped 1 0.00 0.00 0.1% 0.0% + (untracked) 0.02 99.9% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 360.36 0.0% 0.0% + main_timing_calibration 32 354.10 11.07 98.3% 0.0% + flush_timing_buffer 32 5.10 0.16 1.4% 0.0% + timing_overhead 32 0.60 0.02 0.2% 0.0% + (timer overhead subtracted from parent timers) 0.55 0.2% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 11,525.59 0.9% 0.9% + (untracked in python_benchmark_problems) 11,509.17 99.9% 0.9% + (untracked in python_client_execution) 14.71 0.1% 0.0% + (untracked in pre_solution) 1.01 0.0% 0.0% + (untracked in pre_problem) 0.61 0.0% 0.0% + (untracked in post_solution) 0.09 0.0% 0.0% + (untracked in validate_warmups) 0.01 0.0% 0.0% + +TIMING BY TIMER NESTING (legacy view) +----------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +----------------------------------------------------------------------------------------------------------------------------- + python_benchmark_problems 185 1,286,568.71 6,954.43 100.0% + ------------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 1,862 904,610.66 485.83 70.3% 70.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 214 301,368.91 1,408.27 33.3% 23.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 856 236,194.18 275.93 26.1% 18.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_src_co_wait 856 139,026.68 162.41 15.4% 10.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 856 106,019.05 123.85 11.7% 8.2% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 856 69,091.03 80.71 65.2% 5.4% + python_benchpost_library_write 856 26,755.03 31.26 25.2% 2.1% + python_benchpost_naming 856 9,931.36 11.60 9.4% 0.8% + python_benchpost_client_config 856 205.40 0.24 0.2% 0.0% + (untracked) 36.24 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 856 58,429.40 68.26 6.5% 4.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 856 25,692.09 30.01 2.8% 2.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 856 21,607.69 25.24 2.4% 1.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 856 7,968.35 9.31 0.9% 0.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 856 4,549.36 5.31 0.5% 0.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 3,754.95 0.4% 0.3% + ------------------------------------------------------------------------------------------------------------------------- + python_solution_generation 824 315,229.69 382.56 24.5% 24.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_forked_solutions 824 315,154.32 382.47 100.0% 24.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_custom_kernels 824 57.74 0.07 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 13.02 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_fork_permutations 824 4.61 0.01 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_initial_sampling 8 15,265.87 1,908.23 1.2% 1.2% + ------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 856 14,118.81 16.49 1.1% 1.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 856 13,565.73 15.85 96.1% 1.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 856 452.00 0.53 3.2% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_nocache 856 450.45 0.53 99.7% 0.0% + (untracked) 1.54 0.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 87.68 0.6% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 856 13.40 0.02 0.1% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + (untracked) 11,509.17 0.9% 0.9% + ------------------------------------------------------------------------------------------------------------------------- + python_client_execution 32 10,329.62 322.80 0.8% 0.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 9,226.19 89.3% 0.7% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + hip_initialization 32 5,103.79 159.49 55.3% 0.4% + process_exit 32 2,186.50 68.33 23.7% 0.2% + main_problem_factory 32 640.52 20.02 6.9% 0.0% + factory_hip_get_device_properties 4 637.01 159.25 99.5% 0.0% + (untracked) 3.52 0.5% 0.0% + process_startup 32 480.60 15.02 5.2% 0.0% + main_timing_calibration 32 354.10 11.07 3.8% 0.0% + main_teardown_locals 32 150.57 4.71 1.6% 0.0% + data_init_setup 32 97.89 3.06 1.1% 0.0% + code_object_loading 32 92.63 2.89 1.0% 0.0% + library_loading 32 62.38 1.95 0.7% 0.0% + lazy_loading_init 32 43.61 1.36 0.5% 0.0% + main_parse_args 32 10.34 0.32 0.1% 0.0% + python_client_write_run_script 32 1.41 0.04 0.0% 0.0% + reporter_setup 32 1.14 0.04 0.0% 0.0% + listener_setup 32 0.41 0.01 0.0% 0.0% + solution_iterator_setup 32 0.28 0.01 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 1,025.02 9.9% 0.1% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + gpu_input_preparation 40 943.11 23.58 92.0% 0.1% + gpu_input_prep_copy_valid_to_gpu 40 472.41 11.81 50.1% 0.0% + gpu_input_prep_init_cpu_inputs 39 404.45 10.37 42.9% 0.0% + gpu_input_prep_copy_swizzled 4 61.73 15.43 6.5% 0.0% + gpu_input_prep_batched_inputs 40 2.89 0.07 0.3% 0.0% + gpu_input_prep_copy_inputs_d2d 40 1.06 0.03 0.1% 0.0% + gpu_input_prep_constant_inputs 39 0.43 0.01 0.0% 0.0% + gpu_input_prep_convert_to_inputs 40 0.09 0.00 0.0% 0.0% + (untracked) 0.06 0.0% 0.0% + gpu_input_prep_slow_path 40 0.00 0.00 0.0% 0.0% + pre_problem 40 75.46 1.89 7.4% 0.0% + cpu_data_init 1 74.69 74.69 99.0% 0.0% + (untracked) 0.61 0.8% 0.0% + cpu_reference_gemm 1 0.16 0.16 0.2% 0.0% + solve_cpu_slow 1 0.15 0.15 95.3% 0.0% + rotating_buffer_preparation 40 6.41 0.16 0.6% 0.0% + gpu_input_reset 1 0.03 0.03 0.0% 0.0% + gpu_input_reset_copy_inputs_d2d 1 0.03 0.03 93.3% 0.0% + gpu_input_reset_batched_inputs 1 0.00 0.00 1.7% 0.0% + gpu_input_reset_convert_to_inputs 1 0.00 0.00 1.4% 0.0% + gpu_input_reset_copy_swizzled 1 0.00 0.00 0.9% 0.0% + gpu_input_reset_copy_valid_to_gpu 1 0.00 0.00 0.6% 0.0% + gpu_input_reset_slow_path 1 0.00 0.00 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 48.14 0.5% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_solution 585 41.86 0.07 87.0% 0.0% + pre_solution_listener.DataInitialization 585 33.93 0.06 81.1% 0.0% + pre_solution_mx_init_data 130 19.99 0.15 58.9% 0.0% + pre_solution_mx_side_swizzle 8 19.95 2.49 99.8% 0.0% + (untracked) 0.03 0.1% 0.0% + pre_solution_mx_side_cache_hit 252 0.02 0.00 0.1% 0.0% + pre_solution_mx_cpu_sync_no_regen 130 10.19 0.08 30.0% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 130 3.67 0.03 10.8% 0.0% + pre_solution_mx_check 585 0.03 0.00 0.1% 0.0% + (untracked) 0.03 0.1% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 130 0.02 0.00 0.0% 0.0% + pre_solution_listener.BenchmarkTimer 585 3.85 0.01 9.2% 0.0% + pre_solution_listener.ProgressListener 585 1.29 0.00 3.1% 0.0% + pre_solution_listener.AllSolutionsIterator 585 1.16 0.00 2.8% 0.0% + (untracked) 1.01 2.4% 0.0% + pre_solution_listener.MetaResultReporter 585 0.49 0.00 1.2% 0.0% + pre_solution_listener.SynchronizerValidator 585 0.10 0.00 0.2% 0.0% + pre_solution_listener.ReferenceValidator 585 0.02 0.00 0.0% 0.0% + (untracked) 0.02 99.9% 0.0% + pre_solution_ref_skipped 1 0.00 0.00 0.1% 0.0% + pre_solution_listener.HardwareMonitorListener 585 0.02 0.00 0.0% 0.0% + post_solution 585 1.88 0.00 3.9% 0.0% + post_solution_reporting 585 1.12 0.00 59.6% 0.0% + post_solution_log 585 0.60 0.00 32.1% 0.0% + (untracked) 0.09 4.7% 0.0% + post_solution_result_file 585 0.01 0.00 0.8% 0.0% + post_solution_validation 585 0.01 0.00 0.8% 0.0% + post_solution_perf_calc 585 0.01 0.00 0.7% 0.0% + post_solution_sol_advance 585 0.01 0.00 0.7% 0.0% + post_solution_perf_reset 585 0.01 0.00 0.6% 0.0% + run_current_solution_check 585 1.66 0.00 3.4% 0.0% + post_problem 40 1.52 0.04 3.2% 0.0% + finalize_report 32 0.40 0.01 0.8% 0.0% + run_iteration_sync 1 0.18 0.18 0.4% 0.0% + solution_selection 585 0.15 0.00 0.3% 0.0% + problem_report_index 40 0.13 0.00 0.3% 0.0% + warmup_runs 2 0.12 0.06 0.3% 0.0% + problem_teardown 40 0.08 0.00 0.2% 0.0% + kernel_solving 1 0.06 0.06 0.1% 0.0% + warmup_first_sync 1 0.04 0.04 0.1% 0.0% + solution_teardown 585 0.02 0.00 0.0% 0.0% + warmup_events_setup 1 0.02 0.02 0.0% 0.0% + run_iteration_teardown 1 0.01 0.01 0.0% 0.0% + benchmark_runs 1 0.00 0.00 0.0% 0.0% + benchmark_pre_syncs 1 0.00 0.00 36.5% 0.0% + benchmark_post_syncs 1 0.00 0.00 34.5% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked C++ overhead) 14.71 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 9.86 0.1% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 1 9.24 9.24 93.7% 0.0% + validate_warmups_listener.ReferenceValidator 1 9.22 9.22 99.8% 0.0% + validate_gpu_readback 1 7.91 7.91 85.8% 0.0% + validate_result_buffer_alloc 1 1.29 1.29 14.0% 0.0% + validate_element_comparison 1 0.01 0.01 0.1% 0.0% + validate_inputs_copy 1 0.00 0.00 0.0% 0.0% + validate_setup 1 0.00 0.00 0.0% 0.0% + validate_report 1 0.00 0.00 0.0% 0.0% + (untracked) 0.01 0.1% 0.0% + validate_warmups_listener.BenchmarkTimer 1 0.01 0.01 0.1% 0.0% + validate_gpu_sync 1 0.01 0.01 87.1% 0.0% + validate_warmups_listener.HardwareMonitorListener 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.MetaResultReporter 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1 0.00 0.00 0.0% 0.0% + synchronizer_check 1 0.62 0.62 6.3% 0.0% + synchronizer_reduce 1 0.62 0.62 99.6% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 5.71 0.1% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 32 5.10 0.16 89.4% 0.0% + timing_overhead 32 0.60 0.02 10.6% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 856 8,401.41 9.81 0.7% 0.7% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_generate_solutions 32 4,525.43 141.42 0.4% 0.4% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_ga_between_evaluations 8 1,609.22 201.15 0.1% 0.1% + ------------------------------------------------------------------------------------------------------------------------- + python_write_cache 856 918.42 1.07 0.1% 0.1% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_setup 8 35.96 4.50 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_clear_artifacts 24 7.81 0.33 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_read_scores 32 5.63 0.18 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_cache_check 832 0.61 0.00 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_verification 8 0.40 0.05 0.0% 0.0% + + Overlapping Totals (wrap or overlap other timers; not summed into the tree) + --------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 32 10,315.58 322.36 0.8% + main_total 32 7,491.91 234.12 0.6% + problem_total 40 1,083.35 27.08 0.1% + solution_total 585 56.18 0.10 0.0% + run_iteration_total 1 10.34 10.34 0.0% + validate_total 1 9.22 9.22 0.0% + validate_check_results_typed 1 9.22 9.22 0.0% + python_kernel_build_src_co 856 394,368.66 460.71 30.7% + python_kernel_build_src_co.setup 856 191.72 0.22 0.0% + python_kernel_build_src_co.cache_check 856 5,359.01 6.26 0.4% + python_kernel_build_src_co.pch 264 99,216.79 375.82 7.7% + python_kernel_build_src_co.compile 264 287,874.71 1,090.43 22.4% + python_kernel_build_src_co.unbundle 264 1,170.44 4.43 0.1% + python_kernel_build_src_co.move 264 21.46 0.08 0.0% + python_kernel_build_src_co.cache_populate 264 135.07 0.51 0.0% + +----------------------------------------------------------------------------------------------------------------------------- +HIERARCHY VALIDATION: OK (all child phases fit within their parents) + +TIME BREAKDOWN (visual, % of wall clock) +----------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 100.0% ################################################## + BUILD: solution generation & compile 96.5% ################################################ + Solution generation & build (Python) 94.8% ############################################### + python_kernel_compilation 70.3% ################################### + python_kernel_codegen 23.4% ########### + python_kernel_write_assemble 18.4% ######### + python_kernel_build_src_co_wait 10.8% ##### + python_kernel_bench_postprocess 8.2% #### + python_benchpost_lib_construction 5.4% ## + python_benchpost_library_write 2.1% # + python_benchpost_naming 0.8% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_validate 4.5% ## + python_kernel_build_co 2.0% + python_kernel_setup 1.7% + python_kernel_write_helpers 0.6% + python_kernel_bench_setup 0.4% + (untracked) 0.3% + python_solution_generation 24.5% ############ + python_solgen_forked_solutions 24.5% ############ + python_solgen_custom_kernels 0.0% + python_solgen_fork_permutations 0.0% + (untracked) 0.0% + Ductile GA search (Python) 1.7% + python_ductile_initial_sampling 1.2% + python_ductile_generate_solutions 0.4% + python_ductile_ga_between_evaluations 0.1% + python_ductile_setup 0.0% + python_ductile_clear_artifacts 0.0% + python_ductile_read_scores 0.0% + python_ductile_verification 0.0% + SHARED: cache & solution files 1.8% + python_write_solutions 1.1% + python_wsol_dump 1.1% + python_wsol_prepare 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + python_solution_indexing 0.7% + python_write_cache 0.1% + python_cache_check 0.0% + RUN: benchmark client 0.8% + Client launch (startup, setup & teardown) 0.7% + hip_initialization 0.4% + process_exit 0.2% + main_problem_factory 0.0% + factory_hip_get_device_properties 0.0% + (untracked) 0.0% + process_startup 0.0% + main_teardown_locals 0.0% + data_init_setup 0.0% + code_object_loading 0.0% + library_loading 0.0% + lazy_loading_init 0.0% + main_parse_args 0.0% + python_client_write_run_script 0.0% + reporter_setup 0.0% + listener_setup 0.0% + solution_iterator_setup 0.0% + Input data preparation 0.1% + gpu_input_preparation 0.1% + gpu_input_prep_copy_valid_to_gpu 0.0% + gpu_input_prep_init_cpu_inputs 0.0% + gpu_input_prep_copy_swizzled 0.0% + gpu_input_prep_batched_inputs 0.0% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + (untracked) 0.0% + cpu_data_init 0.0% + pre_solution_listener.DataInitialization 0.0% + pre_solution_mx_init_data 0.0% + pre_solution_mx_side_swizzle 0.0% + pre_solution_mx_side_cache_hit 0.0% + (untracked) 0.0% + pre_solution_mx_cpu_sync_no_regen 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 0.0% + pre_solution_mx_check 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 0.0% + (untracked) 0.0% + rotating_buffer_preparation 0.0% + gpu_input_reset 0.0% + gpu_input_reset_copy_inputs_d2d 0.0% + gpu_input_reset_batched_inputs 0.0% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_copy_swizzled 0.0% + gpu_input_reset_copy_valid_to_gpu 0.0% + gpu_input_reset_slow_path 0.0% + Per-solution bookkeeping & reporting 0.0% + pre_solution_listener.BenchmarkTimer 0.0% + run_current_solution_check 0.0% + post_problem 0.0% + pre_solution_listener.ProgressListener 0.0% + pre_solution_listener.AllSolutionsIterator 0.0% + post_solution_reporting 0.0% + post_solution_log 0.0% + pre_solution_listener.MetaResultReporter 0.0% + finalize_report 0.0% + solution_selection 0.0% + problem_report_index 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + problem_teardown 0.0% + solution_teardown 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + run_iteration_teardown 0.0% + post_solution_result_file 0.0% + post_solution_validation 0.0% + post_solution_perf_calc 0.0% + post_solution_sol_advance 0.0% + post_solution_perf_reset 0.0% + Validation 0.0% + validate_warmups_listener.ReferenceValidator 0.0% + validate_gpu_readback 0.0% + validate_result_buffer_alloc 0.0% + validate_element_comparison 0.0% + validate_inputs_copy 0.0% + validate_setup 0.0% + validate_report 0.0% + synchronizer_check 0.0% + synchronizer_reduce 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.DataInitialization 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.ProgressListener 0.0% + validate_warmups_listener.SynchronizerValidator 0.0% + Kernel launch & execution 0.0% + run_iteration_sync 0.0% + warmup_runs 0.0% + kernel_solving 0.0% + warmup_first_sync 0.0% + warmup_events_setup 0.0% + validate_warmups_listener.BenchmarkTimer 0.0% + validate_gpu_sync 0.0% + benchmark_runs 0.0% + benchmark_pre_syncs 0.0% + benchmark_post_syncs 0.0% + Reference GEMM (CPU) 0.0% + cpu_reference_gemm 0.0% + solve_cpu_slow 0.0% + pre_solution_listener.ReferenceValidator 0.0% + pre_solution_ref_skipped 0.0% + (untracked) 0.0% + MEASUREMENT OVERHEAD (instrumentation) 0.0% + main_timing_calibration 0.0% + flush_timing_buffer 0.0% + timing_overhead 0.0% + (timer overhead subtracted from parent timers) 0.0% + UNATTRIBUTED (untracked time & timer overhead) 0.9% + (untracked in python_benchmark_problems) 0.9% + (untracked in python_client_execution) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in post_solution) 0.0% + (untracked in validate_warmups) 0.0% + +KEY INSIGHTS (% of wall clock) +----------------------------------------------------------------------------------------------------------------------------- + Solution generation & build (Python) 1219.84s (94.8%) + SHARED: cache & solution files 23.44s (1.8%) + Ductile GA search (Python) 21.45s (1.7%) + Client launch (startup, setup & teardown) 8.87s (0.7%) + Input data preparation 1.06s (0.1%) + MEASUREMENT OVERHEAD (instrumentation) 0.36s (0.0%) + Per-solution bookkeeping & reporting 0.01s (0.0%) + Validation 0.01s (0.0%) + Kernel launch & execution 0.00s (0.0%) + Reference GEMM (CPU) 0.00s (0.0%) + UNATTRIBUTED (untracked time & timer overhead) 11.53s (0.9%) + + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +----------------------------------------------------------------------------------------------------------------------------- + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + --------------------------------------------------------------------------------------------------------------------------- + 2048 256 128 16 Float8 BFloat16 0.16 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_run/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_run/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..01e8b4120a52 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/after_run/AGGREGATE_analysis.txt @@ -0,0 +1,735 @@ +================================================================================================================================= +TIMING ANALYSIS SUMMARY +================================================================================================================================= + +TIMING BY PURPOSE +--------------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +--------------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 1,028,384.20 100.0% + RUN: benchmark client 891,532.00 86.7% 86.7% + Client launch (startup, setup & teardown) 284,728.58 31.9% 27.7% + hip_initialization 856 127,976.07 149.50 44.9% 12.4% + process_exit 856 67,612.92 78.99 23.7% 6.6% + main_problem_factory 856 34,298.15 40.07 12.0% 3.3% + factory_hip_get_device_properties 192 34,095.91 177.58 99.4% 3.3% + (untracked) 202.25 0.6% 0.0% + process_startup 856 13,627.96 15.92 4.8% 1.3% + code_object_loading 856 10,927.55 12.77 3.8% 1.1% + library_loading 856 9,743.29 11.38 3.4% 0.9% + main_teardown_locals 856 9,697.28 11.33 3.4% 0.9% + data_init_setup 856 9,445.57 11.03 3.3% 0.9% + lazy_loading_init 856 953.97 1.11 0.3% 0.1% + main_parse_args 856 359.25 0.42 0.1% 0.0% + python_client_write_run_script 856 41.59 0.05 0.0% 0.0% + reporter_setup 856 22.85 0.03 0.0% 0.0% + listener_setup 856 12.29 0.01 0.0% 0.0% + solution_iterator_setup 856 9.81 0.01 0.0% 0.0% + Reference GEMM (CPU) 227,584.42 25.5% 22.1% + cpu_reference_gemm 28,596 227,526.92 7.96 100.0% 22.1% + solve_cpu_fast 28,011 226,894.88 8.10 99.7% 22.1% + solve_cpu_slow 592 614.05 1.04 0.3% 0.1% + (untracked) 17.99 0.0% 0.0% + pre_solution_listener.ReferenceValidator 2,355,018 57.51 0.00 0.0% 0.0% + pre_solution_ref_skipped 2,354,328 44.95 0.00 78.2% 0.0% + (untracked) 12.56 21.8% 0.0% + Kernel launch & execution 125,980.76 14.1% 12.3% + warmup_first_sync 2,106,827 93,859.44 0.04 74.5% 9.1% + warmup_runs 4,213,654 25,891.30 0.01 20.6% 2.5% + kernel_solving 2,106,835 4,562.94 0.00 3.6% 0.4% + warmup_events_setup 2,106,835 827.64 0.00 0.7% 0.1% + validate_warmups_listener.BenchmarkTimer 2,106,827 351.37 0.00 0.3% 0.0% + validate_gpu_sync 2,106,827 307.63 0.00 87.6% 0.0% + (untracked) 43.74 12.4% 0.0% + benchmark_runs 2,106,835 267.02 0.00 0.2% 0.0% + benchmark_validate_enqueues 288 104.47 0.36 39.1% 0.0% + validate_enqueues_listener.BenchmarkTimer 288 103.54 0.36 99.1% 0.0% + benchmark_sleep 288 103.48 0.36 99.9% 0.0% + benchmark_event_elapsed_time 8 0.03 0.00 0.0% 0.0% + benchmark_event_destroy 8 0.01 0.00 0.0% 0.0% + (untracked) 0.02 0.0% 0.0% + validate_enqueues_listener.ProgressListener 288 0.78 0.00 0.8% 0.0% + validate_enqueues_listener.MetaResultReporter 288 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.HardwareMonitorListener 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.SynchronizerValidator 288 0.01 0.00 0.0% 0.0% + (untracked) 0.09 0.1% 0.0% + benchmark_post_syncs 2,106,835 74.22 0.00 27.8% 0.0% + benchmark_pre_syncs 2,106,835 52.38 0.00 19.6% 0.0% + gpu_kernel_execution 288 35.88 0.12 13.4% 0.0% + benchmark_events_setup 288 0.06 0.00 0.0% 0.0% + benchmark_events_teardown 288 0.02 0.00 0.0% 0.0% + run_iteration_sync 2,106,835 221.04 0.00 0.2% 0.0% + Input data preparation 122,498.81 13.7% 11.9% + cpu_data_init 28,596 52,377.99 1.83 42.8% 5.1% + gpu_input_reset 2,106,835 46,059.55 0.02 37.6% 4.5% + gpu_input_reset_copy_inputs_d2d 1,473,329 24,899.36 0.02 54.1% 2.4% + gpu_input_reset_batched_inputs 1,473,329 19,254.70 0.01 41.8% 1.9% + gpu_input_reset_fast_reset_output 633,454 1,658.62 0.00 3.6% 0.2% + gpu_input_reset_convert_to_inputs 1,473,329 101.59 0.00 0.2% 0.0% + gpu_input_reset_copy_valid_to_gpu 719,180 95.95 0.00 0.2% 0.0% + gpu_input_reset_slow_path 1,473,329 28.18 0.00 0.1% 0.0% + gpu_input_reset_copy_swizzled 15,719 2.10 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.03 0.01 0.0% 0.0% + gpu_input_reset_constant_inputs 4 0.01 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 4 0.00 0.00 0.0% 0.0% + (untracked) 19.02 0.0% 0.0% + gpu_input_preparation 28,642 20,424.80 0.71 16.7% 2.0% + gpu_input_prep_copy_swizzled 1,151 9,496.79 8.25 46.5% 0.9% + gpu_input_prep_copy_valid_to_gpu 11,446 8,549.51 0.75 41.9% 0.8% + gpu_input_prep_batched_inputs 23,230 1,178.15 0.05 5.8% 0.1% + gpu_input_prep_init_cpu_inputs 41 708.11 17.27 3.5% 0.1% + gpu_input_prep_copy_inputs_d2d 23,230 443.67 0.02 2.2% 0.0% + gpu_input_prep_fast_reset_output 5,403 27.11 0.01 0.1% 0.0% + gpu_input_prep_convert_to_inputs 23,230 10.27 0.00 0.1% 0.0% + gpu_input_prep_slow_path 23,230 2.73 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 44 0.51 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + (untracked) 7.93 0.0% 0.0% + pre_solution_listener.DataInitialization 2,355,018 3,144.42 0.00 2.6% 0.3% + pre_solution_mx_init_data 16,038 1,525.20 0.10 48.5% 0.1% + pre_solution_mx_side_swizzle 2,200 1,521.52 0.69 99.8% 0.1% + pre_solution_mx_side_cache_hit 29,876 2.44 0.00 0.2% 0.0% + (untracked) 1.24 0.1% 0.0% + pre_solution_mx_cpu_sync_no_regen 16,038 1,208.35 0.08 38.4% 0.1% + pre_solution_mx_copy_inputs_d2d_no_regen 16,038 349.86 0.02 11.1% 0.0% + pre_solution_mx_check 2,355,018 49.20 0.00 1.6% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 16,038 1.84 0.00 0.1% 0.0% + (untracked) 9.97 0.3% 0.0% + rotating_buffer_preparation 28,642 492.04 0.02 0.4% 0.0% + Validation 112,494.26 12.6% 10.9% + validate_warmups_listener.ReferenceValidator 2,106,827 100,346.61 0.05 89.2% 9.8% + validate_element_comparison 1,907,339 56,551.97 0.03 56.4% 5.5% + validate_gpu_readback 1,907,339 35,614.28 0.02 35.5% 3.5% + validate_result_buffer_alloc 1,907,339 7,737.11 0.00 7.7% 0.8% + validate_inputs_copy 1,905,787 175.47 0.00 0.2% 0.0% + validate_setup 1,905,883 39.60 0.00 0.0% 0.0% + validate_report 1,907,339 39.04 0.00 0.0% 0.0% + (untracked) 189.15 0.2% 0.0% + synchronizer_check 2,106,835 11,709.85 0.01 10.4% 1.1% + synchronizer_reduce 608,873 11,629.10 0.02 99.3% 1.1% + synchronizer_inactive 1,497,986 28.05 0.00 0.2% 0.0% + (untracked) 52.71 0.5% 0.0% + validate_warmups_listener.MetaResultReporter 2,106,827 116.49 0.00 0.1% 0.0% + validate_warmups_listener.HardwareMonitorListener 2,106,827 67.33 0.00 0.1% 0.0% + validate_warmups_listener.AllSolutionsIterator 2,106,827 64.13 0.00 0.1% 0.0% + validate_warmups_listener.SynchronizerValidator 2,106,827 63.68 0.00 0.1% 0.0% + (untracked) 63.68 100.0% 0.0% + validate_warmups_listener.DataInitialization 2,106,827 63.33 0.00 0.1% 0.0% + validate_warmups_listener.ProgressListener 2,106,827 62.84 0.00 0.1% 0.0% + Per-solution bookkeeping & reporting 13,672.43 1.5% 1.3% + pre_solution_listener.BenchmarkTimer 2,355,018 2,280.84 0.00 16.7% 0.2% + post_solution_log 2,355,018 1,934.47 0.00 14.1% 0.2% + post_solution_reporting 2,355,018 1,812.60 0.00 13.3% 0.2% + run_current_solution_check 2,355,018 1,687.62 0.00 12.3% 0.2% + run_iteration_teardown 2,106,835 1,667.13 0.00 12.2% 0.2% + pre_solution_listener.AllSolutionsIterator 2,355,018 1,225.59 0.00 9.0% 0.1% + pre_solution_listener.MetaResultReporter 2,355,018 836.79 0.00 6.1% 0.1% + pre_solution_listener.ProgressListener 2,355,018 714.19 0.00 5.2% 0.1% + post_solution_validation 2,355,018 530.92 0.00 3.9% 0.1% + solution_selection 2,355,018 401.58 0.00 2.9% 0.0% + pre_solution_listener.SynchronizerValidator 2,355,018 237.01 0.00 1.7% 0.0% + post_problem 28,642 74.45 0.00 0.5% 0.0% + solution_teardown 2,355,018 64.87 0.00 0.5% 0.0% + pre_solution_listener.HardwareMonitorListener 2,355,018 48.70 0.00 0.4% 0.0% + post_solution_sol_advance 2,355,018 46.16 0.00 0.3% 0.0% + post_solution_perf_calc 2,355,018 45.64 0.00 0.3% 0.0% + post_solution_perf_reset 2,355,018 44.36 0.00 0.3% 0.0% + problem_report_index 28,642 13.85 0.00 0.1% 0.0% + problem_teardown 28,642 4.72 0.00 0.0% 0.0% + finalize_report 856 0.95 0.00 0.0% 0.0% + post_solution_result_file 627 0.02 0.00 0.0% 0.0% + Library client (Python) 3,304.79 0.4% 0.3% + python_library_client_create_library 1 3,020.62 3,020.62 91.4% 0.3% + python_library_client_execution 1 277.59 277.59 8.4% 0.0% + python_library_client_subprocess 1 277.11 277.11 99.8% 0.0% + python_library_client_write_run_script 1 0.04 0.04 0.0% 0.0% + (untracked) 0.45 0.2% 0.0% + python_library_client_write_configs 1 5.80 5.80 0.2% 0.0% + python_library_client_setup 1 0.78 0.78 0.0% 0.0% + Library logic (Python) 1,267.93 0.1% 0.1% + python_logic_write 2 1,064.77 532.38 84.0% 0.1% + python_logic_parse_solutions 2 170.41 85.20 13.4% 0.0% + python_logic_analyze 2 32.75 16.38 2.6% 0.0% + BUILD: solution generation & compile 85,606.85 8.3% 8.3% + Solution generation & build (Python) 64,833.54 75.7% 6.3% + python_kernel_compilation 32 64,833.54 2,026.05 100.0% 6.3% + python_kernel_build_src_co 32 41,324.67 1,291.40 63.7% 4.0% + python_kernel_build_src_co.compile 8 41,220.79 5,152.60 99.7% 4.0% + python_kernel_build_src_co.cache_check 32 47.66 1.49 0.1% 0.0% + python_kernel_build_src_co.unbundle 8 45.67 5.71 0.1% 0.0% + python_kernel_build_src_co.cache_populate 8 6.70 0.84 0.0% 0.0% + python_kernel_build_src_co.setup 32 0.82 0.03 0.0% 0.0% + python_kernel_build_src_co.move 8 0.31 0.04 0.0% 0.0% + (untracked) 2.72 0.0% 0.0% + python_kernel_codegen 32 15,972.79 499.15 24.6% 1.6% + python_kernel_write_assemble 32 2,641.41 82.54 4.1% 0.3% + python_kernel_bench_setup 32 1,765.97 55.19 2.7% 0.2% + python_kernel_validate 32 1,316.28 41.13 2.0% 0.1% + python_kernel_build_co 32 688.42 21.51 1.1% 0.1% + python_kernel_write_helpers 32 479.25 14.98 0.7% 0.0% + python_kernel_bench_postprocess 32 470.30 14.70 0.7% 0.0% + python_benchpost_lib_construction 32 335.31 10.48 71.3% 0.0% + python_benchpost_library_write 32 81.61 2.55 17.4% 0.0% + python_benchpost_naming 32 49.05 1.53 10.4% 0.0% + python_benchpost_client_config 32 3.39 0.11 0.7% 0.0% + (untracked) 0.94 0.2% 0.0% + python_kernel_setup 32 131.57 4.11 0.2% 0.0% + (untracked) 42.88 0.1% 0.0% + Ductile GA search (Python) 20,773.31 24.3% 2.0% + python_ductile_initial_sampling 8 14,625.09 1,828.14 70.4% 1.4% + python_ductile_generate_solutions 32 4,612.18 144.13 22.2% 0.4% + python_ductile_ga_between_evaluations 8 1,484.56 185.57 7.1% 0.1% + python_ductile_setup 8 35.59 4.45 0.2% 0.0% + python_ductile_clear_artifacts 32 10.64 0.33 0.1% 0.0% + python_ductile_read_scores 32 4.85 0.15 0.0% 0.0% + python_ductile_verification 8 0.40 0.05 0.0% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 50,077.42 4.9% 4.9% + flush_timing_buffer 857 29,795.24 34.77 59.5% 2.9% + main_timing_calibration 856 10,109.19 11.81 20.2% 1.0% + timing_overhead 856 5,260.68 6.15 10.5% 0.5% + (timer overhead subtracted from parent timers) 4,912.31 9.8% 0.5% + SHARED: cache & solution files 2,590.39 0.3% 0.3% + python_cache_check 832 2,249.35 2.70 86.8% 0.2% + python_write_solutions 856 251.56 0.29 9.7% 0.0% + python_wsol_prepare 856 89.31 0.10 35.5% 0.0% + python_wsol_prepare_cache 824 85.29 0.10 95.5% 0.0% + python_wsol_prepare_nocache 32 0.94 0.03 1.1% 0.0% + (untracked) 3.07 3.4% 0.0% + python_wsol_header 856 85.68 0.10 34.1% 0.0% + python_wsol_dump 32 63.04 1.97 25.1% 0.0% + (untracked) 13.55 5.4% 0.0% + python_write_cache 32 56.19 1.76 2.2% 0.0% + python_solution_indexing 32 33.30 1.04 1.3% 0.0% + UNATTRIBUTED (untracked time & timer overhead) -1,422.47 -0.1% -0.1% + (untracked in python_benchmark_problems) 2,869.74 0.0% 0.3% + (untracked in validate_warmups) 281.75 0.0% 0.0% + (untracked in pre_solution) 249.50 0.0% 0.0% + (untracked in pre_problem) 146.73 0.0% 0.0% + (untracked in python_library_logic) 1.00 0.0% 0.0% + (untracked in python_client_writer) 0.58 0.0% 0.0% + (children exceed benchmark_runs) -39.12 0.0% -0.0% + (children exceed post_solution) -139.33 0.0% -0.0% + (children exceed python_client_execution) -4,793.32 0.0% -0.5% + +TIMING BY TIMER NESTING (legacy view) +--------------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +--------------------------------------------------------------------------------------------------------------------------------- + python_benchmark_problems 185 1,018,897.58 5,507.55 99.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_client_execution 856 927,830.59 1,083.91 91.1% 90.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 347,028.05 37.4% 33.7% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_problem 28,642 280,051.64 9.78 80.7% 27.2% + cpu_reference_gemm 28,596 227,526.92 7.96 81.2% 22.1% + solve_cpu_fast 28,011 226,894.88 8.10 99.7% 22.1% + solve_cpu_slow 592 614.05 1.04 0.3% 0.1% + (untracked) 17.99 0.0% 0.0% + cpu_data_init 28,596 52,377.99 1.83 18.7% 5.1% + (untracked) 146.73 0.1% 0.0% + gpu_input_reset 2,106,835 46,059.55 0.02 13.3% 4.5% + gpu_input_reset_copy_inputs_d2d 1,473,329 24,899.36 0.02 54.1% 2.4% + gpu_input_reset_batched_inputs 1,473,329 19,254.70 0.01 41.8% 1.9% + gpu_input_reset_fast_reset_output 633,454 1,658.62 0.00 3.6% 0.2% + gpu_input_reset_convert_to_inputs 1,473,329 101.59 0.00 0.2% 0.0% + gpu_input_reset_copy_valid_to_gpu 719,180 95.95 0.00 0.2% 0.0% + gpu_input_reset_slow_path 1,473,329 28.18 0.00 0.1% 0.0% + (untracked) 19.02 0.0% 0.0% + gpu_input_reset_copy_swizzled 15,719 2.10 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.03 0.01 0.0% 0.0% + gpu_input_reset_constant_inputs 4 0.01 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 4 0.00 0.00 0.0% 0.0% + gpu_input_preparation 28,642 20,424.80 0.71 5.9% 2.0% + gpu_input_prep_copy_swizzled 1,151 9,496.79 8.25 46.5% 0.9% + gpu_input_prep_copy_valid_to_gpu 11,446 8,549.51 0.75 41.9% 0.8% + gpu_input_prep_batched_inputs 23,230 1,178.15 0.05 5.8% 0.1% + gpu_input_prep_init_cpu_inputs 41 708.11 17.27 3.5% 0.1% + gpu_input_prep_copy_inputs_d2d 23,230 443.67 0.02 2.2% 0.0% + gpu_input_prep_fast_reset_output 5,403 27.11 0.01 0.1% 0.0% + gpu_input_prep_convert_to_inputs 23,230 10.27 0.00 0.1% 0.0% + (untracked) 7.93 0.0% 0.0% + gpu_input_prep_slow_path 23,230 2.73 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 44 0.51 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + rotating_buffer_preparation 28,642 492.04 0.02 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 294,837.77 31.8% 28.7% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + hip_initialization 856 127,976.07 149.50 43.4% 12.4% + process_exit 856 67,612.92 78.99 22.9% 6.6% + main_problem_factory 856 34,298.15 40.07 11.6% 3.3% + factory_hip_get_device_properties 192 34,095.91 177.58 99.4% 3.3% + (untracked) 202.25 0.6% 0.0% + process_startup 856 13,627.96 15.92 4.6% 1.3% + code_object_loading 856 10,927.55 12.77 3.7% 1.1% + main_timing_calibration 856 10,109.19 11.81 3.4% 1.0% + library_loading 856 9,743.29 11.38 3.3% 0.9% + main_teardown_locals 856 9,697.28 11.33 3.3% 0.9% + data_init_setup 856 9,445.57 11.03 3.2% 0.9% + lazy_loading_init 856 953.97 1.11 0.3% 0.1% + main_parse_args 856 359.25 0.42 0.1% 0.0% + python_client_write_run_script 856 41.59 0.05 0.0% 0.0% + reporter_setup 856 22.85 0.03 0.0% 0.0% + listener_setup 856 12.29 0.01 0.0% 0.0% + solution_iterator_setup 856 9.81 0.01 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 142,574.80 15.4% 13.9% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + warmup_first_sync 2,106,827 93,859.44 0.04 65.8% 9.1% + warmup_runs 4,213,654 25,891.30 0.01 18.2% 2.5% + pre_solution 2,355,018 8,794.55 0.00 6.2% 0.9% + pre_solution_listener.DataInitialization 2,355,018 3,144.42 0.00 35.8% 0.3% + pre_solution_mx_init_data 16,038 1,525.20 0.10 48.5% 0.1% + pre_solution_mx_side_swizzle 2,200 1,521.52 0.69 99.8% 0.1% + pre_solution_mx_side_cache_hit 29,876 2.44 0.00 0.2% 0.0% + (untracked) 1.24 0.1% 0.0% + pre_solution_mx_cpu_sync_no_regen 16,038 1,208.35 0.08 38.4% 0.1% + pre_solution_mx_copy_inputs_d2d_no_regen 16,038 349.86 0.02 11.1% 0.0% + pre_solution_mx_check 2,355,018 49.20 0.00 1.6% 0.0% + (untracked) 9.97 0.3% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 16,038 1.84 0.00 0.1% 0.0% + pre_solution_listener.BenchmarkTimer 2,355,018 2,280.84 0.00 25.9% 0.2% + pre_solution_listener.AllSolutionsIterator 2,355,018 1,225.59 0.00 13.9% 0.1% + pre_solution_listener.MetaResultReporter 2,355,018 836.79 0.00 9.5% 0.1% + pre_solution_listener.ProgressListener 2,355,018 714.19 0.00 8.1% 0.1% + (untracked) 249.50 2.8% 0.0% + pre_solution_listener.SynchronizerValidator 2,355,018 237.01 0.00 2.7% 0.0% + pre_solution_listener.ReferenceValidator 2,355,018 57.51 0.00 0.7% 0.0% + pre_solution_ref_skipped 2,354,328 44.95 0.00 78.2% 0.0% + (untracked) 12.56 21.8% 0.0% + pre_solution_listener.HardwareMonitorListener 2,355,018 48.70 0.00 0.6% 0.0% + kernel_solving 2,106,835 4,562.94 0.00 3.2% 0.4% + post_solution 2,355,018 4,274.82 0.00 3.0% 0.4% + post_solution_log 2,355,018 1,934.47 0.00 45.3% 0.2% + post_solution_reporting 2,355,018 1,812.60 0.00 42.4% 0.2% + post_solution_validation 2,355,018 530.92 0.00 12.4% 0.1% + post_solution_sol_advance 2,355,018 46.16 0.00 1.1% 0.0% + post_solution_perf_calc 2,355,018 45.64 0.00 1.1% 0.0% + post_solution_perf_reset 2,355,018 44.36 0.00 1.0% 0.0% + post_solution_result_file 627 0.02 0.00 0.0% 0.0% + run_current_solution_check 2,355,018 1,687.62 0.00 1.2% 0.2% + run_iteration_teardown 2,106,835 1,667.13 0.00 1.2% 0.2% + warmup_events_setup 2,106,835 827.64 0.00 0.6% 0.1% + solution_selection 2,355,018 401.58 0.00 0.3% 0.0% + benchmark_runs 2,106,835 227.90 0.00 0.2% 0.0% + benchmark_validate_enqueues 288 104.47 0.36 45.8% 0.0% + validate_enqueues_listener.BenchmarkTimer 288 103.54 0.36 99.1% 0.0% + benchmark_sleep 288 103.48 0.36 99.9% 0.0% + benchmark_event_elapsed_time 8 0.03 0.00 0.0% 0.0% + (untracked) 0.02 0.0% 0.0% + benchmark_event_destroy 8 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ProgressListener 288 0.78 0.00 0.8% 0.0% + (untracked) 0.09 0.1% 0.0% + validate_enqueues_listener.MetaResultReporter 288 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.HardwareMonitorListener 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 288 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.SynchronizerValidator 288 0.01 0.00 0.0% 0.0% + benchmark_post_syncs 2,106,835 74.22 0.00 32.6% 0.0% + benchmark_pre_syncs 2,106,835 52.38 0.00 23.0% 0.0% + gpu_kernel_execution 288 35.88 0.12 15.7% 0.0% + benchmark_events_setup 288 0.06 0.00 0.0% 0.0% + benchmark_events_teardown 288 0.02 0.00 0.0% 0.0% + run_iteration_sync 2,106,835 221.04 0.00 0.2% 0.0% + post_problem 28,642 74.45 0.00 0.1% 0.0% + solution_teardown 2,355,018 64.87 0.00 0.0% 0.0% + problem_report_index 28,642 13.85 0.00 0.0% 0.0% + problem_teardown 28,642 4.72 0.00 0.0% 0.0% + finalize_report 856 0.95 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 113,127.38 12.2% 11.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 2,106,827 101,417.53 0.05 89.6% 9.9% + validate_warmups_listener.ReferenceValidator 2,106,827 100,346.61 0.05 98.9% 9.8% + validate_element_comparison 1,907,339 56,551.97 0.03 56.4% 5.5% + validate_gpu_readback 1,907,339 35,614.28 0.02 35.5% 3.5% + validate_result_buffer_alloc 1,907,339 7,737.11 0.00 7.7% 0.8% + (untracked) 189.15 0.2% 0.0% + validate_inputs_copy 1,905,787 175.47 0.00 0.2% 0.0% + validate_setup 1,905,883 39.60 0.00 0.0% 0.0% + validate_report 1,907,339 39.04 0.00 0.0% 0.0% + validate_warmups_listener.BenchmarkTimer 2,106,827 351.37 0.00 0.3% 0.0% + validate_gpu_sync 2,106,827 307.63 0.00 87.6% 0.0% + (untracked) 43.74 12.4% 0.0% + (untracked) 281.75 0.3% 0.0% + validate_warmups_listener.MetaResultReporter 2,106,827 116.49 0.00 0.1% 0.0% + validate_warmups_listener.HardwareMonitorListener 2,106,827 67.33 0.00 0.1% 0.0% + validate_warmups_listener.AllSolutionsIterator 2,106,827 64.13 0.00 0.1% 0.0% + validate_warmups_listener.SynchronizerValidator 2,106,827 63.68 0.00 0.1% 0.0% + (untracked) 63.68 100.0% 0.0% + validate_warmups_listener.DataInitialization 2,106,827 63.33 0.00 0.1% 0.0% + validate_warmups_listener.ProgressListener 2,106,827 62.84 0.00 0.1% 0.0% + synchronizer_check 2,106,835 11,709.85 0.01 10.4% 1.1% + synchronizer_reduce 608,873 11,629.10 0.02 99.3% 1.1% + (untracked) 52.71 0.5% 0.0% + synchronizer_inactive 1,497,986 28.05 0.00 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 35,055.92 3.8% 3.4% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 857 29,795.24 34.77 85.0% 2.9% + timing_overhead 856 5,260.68 6.15 15.0% 0.5% + ----------------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 32 64,833.54 2,026.05 6.4% 6.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_src_co 32 41,324.67 1,291.40 63.7% 4.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_kernel_build_src_co.compile 8 41,220.79 5,152.60 99.7% 4.0% + python_kernel_build_src_co.cache_check 32 47.66 1.49 0.1% 0.0% + python_kernel_build_src_co.unbundle 8 45.67 5.71 0.1% 0.0% + python_kernel_build_src_co.cache_populate 8 6.70 0.84 0.0% 0.0% + (untracked) 2.72 0.0% 0.0% + python_kernel_build_src_co.setup 32 0.82 0.03 0.0% 0.0% + python_kernel_build_src_co.move 8 0.31 0.04 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 32 15,972.79 499.15 24.6% 1.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 32 2,641.41 82.54 4.1% 0.3% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 32 1,765.97 55.19 2.7% 0.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 32 1,316.28 41.13 2.0% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 32 688.42 21.51 1.1% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 32 479.25 14.98 0.7% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 32 470.30 14.70 0.7% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 32 335.31 10.48 71.3% 0.0% + python_benchpost_library_write 32 81.61 2.55 17.4% 0.0% + python_benchpost_naming 32 49.05 1.53 10.4% 0.0% + python_benchpost_client_config 32 3.39 0.11 0.7% 0.0% + (untracked) 0.94 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 32 131.57 4.11 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 42.88 0.1% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_initial_sampling 8 14,625.09 1,828.14 1.4% 1.4% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_generate_solutions 32 4,612.18 144.13 0.5% 0.4% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 2,869.74 0.3% 0.3% + ----------------------------------------------------------------------------------------------------------------------------- + python_cache_check 832 2,249.35 2.70 0.2% 0.2% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_ga_between_evaluations 8 1,484.56 185.57 0.1% 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 856 251.56 0.29 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 856 89.31 0.10 35.5% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_cache 824 85.29 0.10 95.5% 0.0% + (untracked) 3.07 3.4% 0.0% + python_wsol_prepare_nocache 32 0.94 0.03 1.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 856 85.68 0.10 34.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 32 63.04 1.97 25.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 13.55 5.4% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_write_cache 32 56.19 1.76 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_setup 8 35.59 4.45 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 32 33.30 1.04 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_clear_artifacts 32 10.64 0.33 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_read_scores 32 4.85 0.15 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_verification 8 0.40 0.05 0.0% 0.0% + =============================================================================================================================== + python_client_writer 1 3,305.37 3,305.37 0.3% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_create_library 1 3,020.62 3,020.62 91.4% 0.3% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_execution 1 277.59 277.59 8.4% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_subprocess 1 277.11 277.11 99.8% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 0.45 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_write_run_script 1 0.04 0.04 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_write_configs 1 5.80 5.80 0.2% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_setup 1 0.78 0.78 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 0.58 0.0% 0.0% + =============================================================================================================================== + python_library_logic 2 1,268.93 634.46 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_write 2 1,064.77 532.38 83.9% 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_parse_solutions 2 170.41 85.20 13.4% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_analyze 2 32.75 16.38 2.6% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 1.00 0.1% 0.0% + + Overlapping Totals (wrap other timers; not summed into the tree) + ------------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 856 932,308.02 1,089.14 90.7% + main_total 856 811,544.39 948.07 78.9% + problem_total 28,642 607,652.46 21.22 59.1% + solution_total 2,355,018 306,431.61 0.13 29.8% + run_iteration_total 2,106,835 289,194.98 0.14 28.1% + validate_total 1,905,883 100,395.19 0.05 9.8% + validate_check_results_typed 1,907,339 100,168.89 0.05 9.7% + +--------------------------------------------------------------------------------------------------------------------------------- +HIERARCHY VALIDATION WARNINGS +--------------------------------------------------------------------------------------------------------------------------------- + WARNING: Child phases of python_client_execution (936,062.04 ms) exceed parent (932,742.66 ms) by 3,319.38 ms (0.4%) + +TIME BREAKDOWN (visual, % of wall clock) +--------------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 100.0% ################################################## + RUN: benchmark client 86.7% ########################################### + Client launch (startup, setup & teardown) 27.7% ############# + hip_initialization 12.4% ###### + process_exit 6.6% ### + main_problem_factory 3.3% # + factory_hip_get_device_properties 3.3% # + (untracked) 0.0% + process_startup 1.3% + code_object_loading 1.1% + library_loading 0.9% + main_teardown_locals 0.9% + data_init_setup 0.9% + lazy_loading_init 0.1% + main_parse_args 0.0% + python_client_write_run_script 0.0% + reporter_setup 0.0% + listener_setup 0.0% + solution_iterator_setup 0.0% + Reference GEMM (CPU) 22.1% ########### + cpu_reference_gemm 22.1% ########### + solve_cpu_fast 22.1% ########### + solve_cpu_slow 0.1% + (untracked) 0.0% + pre_solution_listener.ReferenceValidator 0.0% + pre_solution_ref_skipped 0.0% + (untracked) 0.0% + Kernel launch & execution 12.3% ###### + warmup_first_sync 9.1% #### + warmup_runs 2.5% # + kernel_solving 0.4% + warmup_events_setup 0.1% + validate_warmups_listener.BenchmarkTimer 0.0% + validate_gpu_sync 0.0% + (untracked) 0.0% + benchmark_runs 0.0% + benchmark_validate_enqueues 0.0% + validate_enqueues_listener.BenchmarkTimer 0.0% + benchmark_sleep 0.0% + benchmark_event_elapsed_time 0.0% + benchmark_event_destroy 0.0% + (untracked) 0.0% + validate_enqueues_listener.ProgressListener 0.0% + validate_enqueues_listener.MetaResultReporter 0.0% + validate_enqueues_listener.DataInitialization 0.0% + validate_enqueues_listener.ReferenceValidator 0.0% + validate_enqueues_listener.HardwareMonitorListener 0.0% + validate_enqueues_listener.AllSolutionsIterator 0.0% + validate_enqueues_listener.SynchronizerValidator 0.0% + (untracked) 0.0% + benchmark_post_syncs 0.0% + benchmark_pre_syncs 0.0% + gpu_kernel_execution 0.0% + benchmark_events_setup 0.0% + benchmark_events_teardown 0.0% + run_iteration_sync 0.0% + Input data preparation 11.9% ##### + cpu_data_init 5.1% ## + gpu_input_reset 4.5% ## + gpu_input_reset_copy_inputs_d2d 2.4% # + gpu_input_reset_batched_inputs 1.9% + gpu_input_reset_fast_reset_output 0.2% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_copy_valid_to_gpu 0.0% + gpu_input_reset_slow_path 0.0% + gpu_input_reset_copy_swizzled 0.0% + gpu_input_reset_rotating_init 0.0% + gpu_input_reset_constant_inputs 0.0% + gpu_input_reset_init_cpu_inputs 0.0% + (untracked) 0.0% + gpu_input_preparation 2.0% + gpu_input_prep_copy_swizzled 0.9% + gpu_input_prep_copy_valid_to_gpu 0.8% + gpu_input_prep_batched_inputs 0.1% + gpu_input_prep_init_cpu_inputs 0.1% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_fast_reset_output 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_rotating_init 0.0% + (untracked) 0.0% + pre_solution_listener.DataInitialization 0.3% + pre_solution_mx_init_data 0.1% + pre_solution_mx_side_swizzle 0.1% + pre_solution_mx_side_cache_hit 0.0% + (untracked) 0.0% + pre_solution_mx_cpu_sync_no_regen 0.1% + pre_solution_mx_copy_inputs_d2d_no_regen 0.0% + pre_solution_mx_check 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 0.0% + (untracked) 0.0% + rotating_buffer_preparation 0.0% + Validation 10.9% ##### + validate_warmups_listener.ReferenceValidator 9.8% #### + validate_element_comparison 5.5% ## + validate_gpu_readback 3.5% # + validate_result_buffer_alloc 0.8% + validate_inputs_copy 0.0% + validate_setup 0.0% + validate_report 0.0% + (untracked) 0.0% + synchronizer_check 1.1% + synchronizer_reduce 1.1% + synchronizer_inactive 0.0% + (untracked) 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.SynchronizerValidator 0.0% + (untracked) 0.0% + validate_warmups_listener.DataInitialization 0.0% + validate_warmups_listener.ProgressListener 0.0% + Per-solution bookkeeping & reporting 1.3% + pre_solution_listener.BenchmarkTimer 0.2% + post_solution_log 0.2% + post_solution_reporting 0.2% + run_current_solution_check 0.2% + run_iteration_teardown 0.2% + pre_solution_listener.AllSolutionsIterator 0.1% + pre_solution_listener.MetaResultReporter 0.1% + pre_solution_listener.ProgressListener 0.1% + post_solution_validation 0.1% + solution_selection 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + post_problem 0.0% + solution_teardown 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + post_solution_sol_advance 0.0% + post_solution_perf_calc 0.0% + post_solution_perf_reset 0.0% + problem_report_index 0.0% + problem_teardown 0.0% + finalize_report 0.0% + post_solution_result_file 0.0% + Library client (Python) 0.3% + python_library_client_create_library 0.3% + python_library_client_execution 0.0% + python_library_client_subprocess 0.0% + python_library_client_write_run_script 0.0% + (untracked) 0.0% + python_library_client_write_configs 0.0% + python_library_client_setup 0.0% + Library logic (Python) 0.1% + python_logic_write 0.1% + python_logic_parse_solutions 0.0% + python_logic_analyze 0.0% + BUILD: solution generation & compile 8.3% #### + Solution generation & build (Python) 6.3% ### + python_kernel_compilation 6.3% ### + python_kernel_build_src_co 4.0% ## + python_kernel_build_src_co.compile 4.0% ## + python_kernel_build_src_co.cache_check 0.0% + python_kernel_build_src_co.unbundle 0.0% + python_kernel_build_src_co.cache_populate 0.0% + python_kernel_build_src_co.setup 0.0% + python_kernel_build_src_co.move 0.0% + (untracked) 0.0% + python_kernel_codegen 1.6% + python_kernel_write_assemble 0.3% + python_kernel_bench_setup 0.2% + python_kernel_validate 0.1% + python_kernel_build_co 0.1% + python_kernel_write_helpers 0.0% + python_kernel_bench_postprocess 0.0% + python_benchpost_lib_construction 0.0% + python_benchpost_library_write 0.0% + python_benchpost_naming 0.0% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_setup 0.0% + (untracked) 0.0% + Ductile GA search (Python) 2.0% # + python_ductile_initial_sampling 1.4% + python_ductile_generate_solutions 0.4% + python_ductile_ga_between_evaluations 0.1% + python_ductile_setup 0.0% + python_ductile_clear_artifacts 0.0% + python_ductile_read_scores 0.0% + python_ductile_verification 0.0% + MEASUREMENT OVERHEAD (instrumentation) 4.9% ## + flush_timing_buffer 2.9% # + main_timing_calibration 1.0% + timing_overhead 0.5% + (timer overhead subtracted from parent timers) 0.5% + SHARED: cache & solution files 0.3% + python_cache_check 0.2% + python_write_solutions 0.0% + python_wsol_prepare 0.0% + python_wsol_prepare_cache 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + python_wsol_dump 0.0% + (untracked) 0.0% + python_write_cache 0.0% + python_solution_indexing 0.0% + UNATTRIBUTED (untracked time & timer overhead) -0.1% + (untracked in python_benchmark_problems) 0.3% + (untracked in validate_warmups) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in python_library_logic) 0.0% + (untracked in python_client_writer) 0.0% + (children exceed benchmark_runs) -0.0% + (children exceed post_solution) -0.0% + (children exceed python_client_execution) -0.5% + +KEY INSIGHTS (% of wall clock) +--------------------------------------------------------------------------------------------------------------------------------- + Client launch (startup, setup & teardown) 284.73s (27.7%) + Reference GEMM (CPU) 227.58s (22.1%) + Kernel launch & execution 125.98s (12.3%) + Input data preparation 122.50s (11.9%) + Validation 112.49s (10.9%) + Solution generation & build (Python) 64.83s (6.3%) + MEASUREMENT OVERHEAD (instrumentation) 45.17s (4.4%) + Ductile GA search (Python) 20.77s (2.0%) + Per-solution bookkeeping & reporting 13.67s (1.3%) + Library client (Python) 3.30s (0.3%) + SHARED: cache & solution files 2.59s (0.3%) + Library logic (Python) 1.27s (0.1%) + UNATTRIBUTED (untracked time & timer overhead) -1.42s (-0.1%) + + CPU reference / GPU execution ratio: 6341.3x + >>> CPU reference > 10x GPU execution time! + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +--------------------------------------------------------------------------------------------------------------------------------- + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + ------------------------------------------------------------------------------------------------------------------------------- + 43776 10624 6400 1 Float8 BFloat16 19546.98 0.00 + 13056 7680 21504 1 Float8 BFloat16 17040.82 0.00 + 71936 4096 7168 1 Float8 BFloat16 15624.38 0.00 + 13696 15488 8448 1 Float8 BFloat16 14073.45 0.00 + 14336 10880 9472 1 Float8 BFloat16 10054.99 0.00 + 9728 9600 13824 1 Float8 BFloat16 9428.36 0.00 + 173312 3200 2560 1 Float8 BFloat16 8879.09 0.00 + 14336 10880 9472 1 Float8 BFloat16 7652.50 0.00 + 9728 9600 13824 1 Float8 BFloat16 7205.78 0.00 + 209536 2048 1280 1 Float8 BFloat16 5264.93 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_build/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_build/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..ba79d2b75150 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_build/AGGREGATE_analysis.txt @@ -0,0 +1,595 @@ +============================================================================================================================= +TIMING ANALYSIS SUMMARY +============================================================================================================================= + +TIMING BY PURPOSE +----------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +----------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 2,654,787.19 100.0% + BUILD: solution generation & compile 2,605,637.61 98.1% 98.1% + Solution generation & build (Python) 2,581,934.64 99.1% 97.3% + python_kernel_compilation 855 2,260,356.50 2,643.69 87.5% 85.1% + python_kernel_build_src_co 855 1,013,356.87 1,185.21 44.8% 38.2% + python_kernel_build_src_co.compile 263 1,010,494.31 3,842.18 99.7% 38.1% + python_kernel_build_src_co.unbundle 263 1,350.69 5.14 0.1% 0.1% + python_kernel_build_src_co.cache_check 855 1,203.64 1.41 0.1% 0.0% + python_kernel_build_src_co.cache_populate 263 175.68 0.67 0.0% 0.0% + python_kernel_build_src_co.setup 855 25.18 0.03 0.0% 0.0% + python_kernel_build_src_co.move 263 14.10 0.05 0.0% 0.0% + (untracked) 93.27 0.0% 0.0% + python_kernel_codegen 855 615,536.08 719.93 27.2% 23.2% + python_kernel_write_assemble 855 253,795.98 296.84 11.2% 9.6% + python_kernel_bench_setup 855 132,219.33 154.64 5.8% 5.0% + python_kernel_bench_postprocess 855 105,243.40 123.09 4.7% 4.0% + python_benchpost_lib_construction 855 72,859.94 85.22 69.2% 2.7% + python_benchpost_library_write 855 22,779.07 26.64 21.6% 0.9% + python_benchpost_naming 855 9,378.78 10.97 8.9% 0.4% + python_benchpost_client_config 855 194.33 0.23 0.2% 0.0% + (untracked) 31.28 0.0% 0.0% + python_kernel_validate 855 82,550.97 96.55 3.7% 3.1% + python_kernel_build_co 855 22,054.87 25.80 1.0% 0.8% + python_kernel_setup 855 20,891.95 24.44 0.9% 0.8% + python_kernel_write_helpers 855 10,823.20 12.66 0.5% 0.4% + (untracked) 3,883.85 0.2% 0.1% + python_solution_generation 823 321,578.14 390.74 12.5% 12.1% + python_solgen_forked_solutions 823 321,501.91 390.65 100.0% 12.1% + python_solgen_custom_kernels 823 59.09 0.07 0.0% 0.0% + python_solgen_fork_permutations 823 4.51 0.01 0.0% 0.0% + (untracked) 12.63 0.0% 0.0% + Ductile GA search (Python) 23,702.97 0.9% 0.9% + python_ductile_initial_sampling 8 17,418.90 2,177.36 73.5% 0.7% + python_ductile_generate_solutions 32 4,866.91 152.09 20.5% 0.2% + python_ductile_ga_between_evaluations 8 1,364.98 170.62 5.8% 0.1% + python_ductile_setup 8 29.94 3.74 0.1% 0.0% + python_ductile_clear_artifacts 24 17.63 0.73 0.1% 0.0% + python_ductile_read_scores 32 4.30 0.13 0.0% 0.0% + python_ductile_verification 8 0.30 0.04 0.0% 0.0% + SHARED: cache & solution files 23,204.09 0.9% 0.9% + python_write_solutions 855 14,089.48 16.48 60.7% 0.5% + python_wsol_dump 855 13,420.73 15.70 95.3% 0.5% + python_wsol_prepare 855 528.55 0.62 3.8% 0.0% + python_wsol_prepare_nocache 855 527.16 0.62 99.7% 0.0% + (untracked) 1.39 0.3% 0.0% + python_wsol_header 855 11.83 0.01 0.1% 0.0% + (untracked) 128.36 0.9% 0.0% + python_solution_indexing 855 8,355.06 9.77 36.0% 0.3% + python_write_cache 855 758.98 0.89 3.3% 0.0% + python_cache_check 831 0.58 0.00 0.0% 0.0% + RUN: benchmark client 14,327.88 0.5% 0.5% + Client launch (startup, setup & teardown) 9,447.60 65.9% 0.4% + hip_initialization 32 5,005.78 156.43 53.0% 0.2% + process_exit 32 2,438.60 76.21 25.8% 0.1% + main_problem_factory 32 629.47 19.67 6.7% 0.0% + factory_hip_get_device_properties 4 625.92 156.48 99.4% 0.0% + (untracked) 3.55 0.6% 0.0% + main_teardown_locals 32 589.74 18.43 6.2% 0.0% + process_startup 32 467.09 14.60 4.9% 0.0% + data_init_setup 32 98.04 3.06 1.0% 0.0% + code_object_loading 32 97.55 3.05 1.0% 0.0% + library_loading 32 64.25 2.01 0.7% 0.0% + lazy_loading_init 32 42.62 1.33 0.5% 0.0% + main_parse_args 32 10.48 0.33 0.1% 0.0% + reporter_setup 32 1.66 0.05 0.0% 0.0% + python_client_write_run_script 32 1.58 0.05 0.0% 0.0% + listener_setup 32 0.46 0.01 0.0% 0.0% + solution_iterator_setup 32 0.27 0.01 0.0% 0.0% + Input data preparation 4,796.72 33.5% 0.2% + gpu_input_preparation 40 3,536.05 88.40 73.7% 0.1% + gpu_input_prep_init_cpu_inputs 39 2,097.58 53.78 59.3% 0.1% + gpu_input_prep_copy_valid_to_gpu 40 1,349.34 33.73 38.2% 0.1% + gpu_input_prep_copy_swizzled 4 85.02 21.25 2.4% 0.0% + gpu_input_prep_batched_inputs 40 2.57 0.06 0.1% 0.0% + gpu_input_prep_copy_inputs_d2d 40 0.98 0.02 0.0% 0.0% + gpu_input_prep_constant_inputs 39 0.43 0.01 0.0% 0.0% + gpu_input_prep_convert_to_inputs 40 0.08 0.00 0.0% 0.0% + gpu_input_prep_slow_path 40 0.00 0.00 0.0% 0.0% + (untracked) 0.05 0.0% 0.0% + pre_solution_listener.DataInitialization 585 1,120.43 1.92 23.4% 0.0% + pre_solution_mx_init_data 130 1,000.38 7.70 89.3% 0.0% + pre_solution_mx_side_regenerate 8 1,000.33 125.04 100.0% 0.0% + pre_solution_mx_side_cache_hit 252 0.02 0.00 0.0% 0.0% + (untracked) 0.03 0.0% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 126 103.32 0.82 9.2% 0.0% + pre_solution_mx_cpu_sync_no_regen 126 8.60 0.07 0.8% 0.0% + pre_solution_mx_copy_valid_to_gpu_after_regen 4 3.38 0.85 0.3% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 126 3.22 0.03 0.3% 0.0% + pre_solution_mx_cpu_sync_after_regen 4 1.32 0.33 0.1% 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 4 0.14 0.03 0.0% 0.0% + pre_solution_mx_check 585 0.03 0.00 0.0% 0.0% + (untracked) 0.03 0.0% 0.0% + cpu_data_init 1 132.63 132.63 2.8% 0.0% + rotating_buffer_preparation 40 5.84 0.15 0.1% 0.0% + gpu_input_reset 1 1.76 1.76 0.0% 0.0% + gpu_input_reset_copy_valid_to_gpu 1 0.84 0.84 47.8% 0.0% + gpu_input_reset_copy_swizzled 1 0.83 0.83 46.9% 0.0% + gpu_input_reset_batched_inputs 1 0.06 0.06 3.6% 0.0% + gpu_input_reset_copy_inputs_d2d 1 0.03 0.03 1.7% 0.0% + gpu_input_reset_convert_to_inputs 1 0.00 0.00 0.0% 0.0% + gpu_input_reset_slow_path 1 0.00 0.00 0.0% 0.0% + Per-solution bookkeeping & reporting 67.12 0.5% 0.0% + pre_solution_listener.MetaResultReporter 585 36.70 0.06 54.7% 0.0% + pre_solution_listener.ProgressListener 585 15.18 0.03 22.6% 0.0% + post_solution_log 585 5.47 0.01 8.2% 0.0% + pre_solution_listener.BenchmarkTimer 585 3.41 0.01 5.1% 0.0% + run_current_solution_check 585 1.61 0.00 2.4% 0.0% + post_problem 40 1.61 0.04 2.4% 0.0% + pre_solution_listener.AllSolutionsIterator 585 1.11 0.00 1.6% 0.0% + post_solution_reporting 585 0.99 0.00 1.5% 0.0% + finalize_report 32 0.34 0.01 0.5% 0.0% + post_solution_perf_calc 585 0.20 0.00 0.3% 0.0% + solution_selection 585 0.14 0.00 0.2% 0.0% + problem_report_index 40 0.13 0.00 0.2% 0.0% + pre_solution_listener.SynchronizerValidator 585 0.10 0.00 0.1% 0.0% + problem_teardown 40 0.04 0.00 0.1% 0.0% + solution_teardown 585 0.02 0.00 0.0% 0.0% + post_solution_result_file 585 0.02 0.00 0.0% 0.0% + pre_solution_listener.HardwareMonitorListener 585 0.01 0.00 0.0% 0.0% + post_solution_validation 585 0.01 0.00 0.0% 0.0% + post_solution_sol_advance 585 0.01 0.00 0.0% 0.0% + post_solution_perf_reset 585 0.01 0.00 0.0% 0.0% + run_iteration_teardown 1 0.01 0.01 0.0% 0.0% + Reference GEMM (CPU) 14.60 0.1% 0.0% + cpu_reference_gemm 1 13.61 13.61 93.2% 0.0% + solve_cpu_slow 1 13.61 13.61 100.0% 0.0% + pre_solution_listener.ReferenceValidator 585 0.99 0.00 6.8% 0.0% + cpu_reference_gemm_per_solution_inputs_changed 1 0.97 0.97 98.1% 0.0% + solve_cpu_slow_per_solution_inputs_changed 1 0.97 0.97 100.0% 0.0% + (untracked) 0.02 1.9% 0.0% + Validation 1.69 0.0% 0.0% + validate_warmups_listener.ReferenceValidator 1 1.18 1.18 69.7% 0.0% + validate_result_buffer_alloc 1 0.87 0.87 73.5% 0.0% + validate_gpu_readback 1 0.30 0.30 25.7% 0.0% + validate_element_comparison 1 0.01 0.01 0.8% 0.0% + validate_inputs_copy 1 0.00 0.00 0.1% 0.0% + validate_report 1 0.00 0.00 0.0% 0.0% + validate_setup 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1 0.51 0.51 30.3% 0.0% + validate_synchronizer_check 1 0.51 0.51 100.0% 0.0% + validate_synchronizer_readback 1 0.34 0.34 66.6% 0.0% + validate_synchronizer_scan 1 0.17 0.17 33.4% 0.0% + validate_warmups_listener.MetaResultReporter 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1 0.00 0.00 0.0% 0.0% + Kernel launch & execution 0.16 0.0% 0.0% + warmup_runs 2 0.07 0.04 46.1% 0.0% + kernel_solving 1 0.06 0.06 39.0% 0.0% + warmup_events_setup 1 0.02 0.02 12.0% 0.0% + validate_warmups_listener.BenchmarkTimer 1 0.00 0.00 2.6% 0.0% + validate_gpu_sync 1 0.00 0.00 100.0% 0.0% + benchmark_runs 1 0.00 0.00 0.2% 0.0% + benchmark_pre_syncs 1 0.00 0.00 50.0% 0.0% + benchmark_post_syncs 1 0.00 0.00 50.0% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 362.43 0.0% 0.0% + main_timing_calibration 32 356.49 11.14 98.4% 0.0% + flush_timing_buffer 32 4.73 0.15 1.3% 0.0% + (timer overhead subtracted from parent timers) 0.61 0.2% 0.0% + timing_overhead 32 0.60 0.02 0.2% 0.0% + UNATTRIBUTED (untracked time & timer overhead) 11,255.17 0.4% 0.4% + (untracked in python_benchmark_problems) 11,241.68 99.9% 0.4% + (untracked in python_client_execution) 11.94 0.1% 0.0% + (untracked in pre_solution) 0.89 0.0% 0.0% + (untracked in pre_problem) 0.59 0.0% 0.0% + (untracked in post_solution) 0.07 0.0% 0.0% + +TIMING BY TIMER NESTING (legacy view) +----------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +----------------------------------------------------------------------------------------------------------------------------- + python_benchmark_problems 184 2,654,786.60 14,428.19 100.0% + ------------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 855 2,260,356.50 2,643.69 85.1% 85.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_src_co 855 1,013,356.87 1,185.21 44.8% 38.2% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_kernel_build_src_co.compile 263 1,010,494.31 3,842.18 99.7% 38.1% + python_kernel_build_src_co.unbundle 263 1,350.69 5.14 0.1% 0.1% + python_kernel_build_src_co.cache_check 855 1,203.64 1.41 0.1% 0.0% + python_kernel_build_src_co.cache_populate 263 175.68 0.67 0.0% 0.0% + (untracked) 93.27 0.0% 0.0% + python_kernel_build_src_co.setup 855 25.18 0.03 0.0% 0.0% + python_kernel_build_src_co.move 263 14.10 0.05 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 855 615,536.08 719.93 27.2% 23.2% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 855 253,795.98 296.84 11.2% 9.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 855 132,219.33 154.64 5.8% 5.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 855 105,243.40 123.09 4.7% 4.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 855 72,859.94 85.22 69.2% 2.7% + python_benchpost_library_write 855 22,779.07 26.64 21.6% 0.9% + python_benchpost_naming 855 9,378.78 10.97 8.9% 0.4% + python_benchpost_client_config 855 194.33 0.23 0.2% 0.0% + (untracked) 31.28 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 855 82,550.97 96.55 3.7% 3.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 855 22,054.87 25.80 1.0% 0.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 855 20,891.95 24.44 0.9% 0.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 855 10,823.20 12.66 0.5% 0.4% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 3,883.85 0.2% 0.1% + ------------------------------------------------------------------------------------------------------------------------- + python_solution_generation 823 321,578.14 390.74 12.1% 12.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_forked_solutions 823 321,501.91 390.65 100.0% 12.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_custom_kernels 823 59.09 0.07 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 12.63 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_solgen_fork_permutations 823 4.51 0.01 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_initial_sampling 8 17,418.90 2,177.36 0.7% 0.7% + ------------------------------------------------------------------------------------------------------------------------- + python_client_execution 32 14,703.21 459.48 0.6% 0.6% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 9,804.08 66.7% 0.4% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + hip_initialization 32 5,005.78 156.43 51.1% 0.2% + process_exit 32 2,438.60 76.21 24.9% 0.1% + main_problem_factory 32 629.47 19.67 6.4% 0.0% + factory_hip_get_device_properties 4 625.92 156.48 99.4% 0.0% + (untracked) 3.55 0.6% 0.0% + main_teardown_locals 32 589.74 18.43 6.0% 0.0% + process_startup 32 467.09 14.60 4.8% 0.0% + main_timing_calibration 32 356.49 11.14 3.6% 0.0% + data_init_setup 32 98.04 3.06 1.0% 0.0% + code_object_loading 32 97.55 3.05 1.0% 0.0% + library_loading 32 64.25 2.01 0.7% 0.0% + lazy_loading_init 32 42.62 1.33 0.4% 0.0% + main_parse_args 32 10.48 0.33 0.1% 0.0% + reporter_setup 32 1.66 0.05 0.0% 0.0% + python_client_write_run_script 32 1.58 0.05 0.0% 0.0% + listener_setup 32 0.46 0.01 0.0% 0.0% + solution_iterator_setup 32 0.27 0.01 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 3,690.49 25.1% 0.1% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + gpu_input_preparation 40 3,536.05 88.40 95.8% 0.1% + gpu_input_prep_init_cpu_inputs 39 2,097.58 53.78 59.3% 0.1% + gpu_input_prep_copy_valid_to_gpu 40 1,349.34 33.73 38.2% 0.1% + gpu_input_prep_copy_swizzled 4 85.02 21.25 2.4% 0.0% + gpu_input_prep_batched_inputs 40 2.57 0.06 0.1% 0.0% + gpu_input_prep_copy_inputs_d2d 40 0.98 0.02 0.0% 0.0% + gpu_input_prep_constant_inputs 39 0.43 0.01 0.0% 0.0% + gpu_input_prep_convert_to_inputs 40 0.08 0.00 0.0% 0.0% + (untracked) 0.05 0.0% 0.0% + gpu_input_prep_slow_path 40 0.00 0.00 0.0% 0.0% + pre_problem 40 146.83 3.67 4.0% 0.0% + cpu_data_init 1 132.63 132.63 90.3% 0.0% + cpu_reference_gemm 1 13.62 13.62 9.3% 0.0% + solve_cpu_slow 1 13.61 13.61 99.9% 0.0% + (untracked) 0.59 0.4% 0.0% + rotating_buffer_preparation 40 5.84 0.15 0.2% 0.0% + gpu_input_reset 1 1.76 1.76 0.0% 0.0% + gpu_input_reset_copy_valid_to_gpu 1 0.84 0.84 47.8% 0.0% + gpu_input_reset_copy_swizzled 1 0.83 0.83 46.9% 0.0% + gpu_input_reset_batched_inputs 1 0.06 0.06 3.6% 0.0% + gpu_input_reset_copy_inputs_d2d 1 0.03 0.03 1.7% 0.0% + gpu_input_reset_convert_to_inputs 1 0.00 0.00 0.0% 0.0% + gpu_input_reset_slow_path 1 0.00 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 1,189.65 8.1% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_solution 585 1,178.81 2.02 99.1% 0.0% + pre_solution_listener.DataInitialization 585 1,120.43 1.92 95.0% 0.0% + pre_solution_mx_init_data 130 1,000.38 7.70 89.3% 0.0% + pre_solution_mx_side_regenerate 8 1,000.33 125.04 100.0% 0.0% + (untracked) 0.03 0.0% 0.0% + pre_solution_mx_side_cache_hit 252 0.02 0.00 0.0% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 126 103.32 0.82 9.2% 0.0% + pre_solution_mx_cpu_sync_no_regen 126 8.60 0.07 0.8% 0.0% + pre_solution_mx_copy_valid_to_gpu_after_regen 4 3.38 0.85 0.3% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 126 3.22 0.03 0.3% 0.0% + pre_solution_mx_cpu_sync_after_regen 4 1.32 0.33 0.1% 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 4 0.14 0.03 0.0% 0.0% + (untracked) 0.03 0.0% 0.0% + pre_solution_mx_check 585 0.03 0.00 0.0% 0.0% + pre_solution_listener.MetaResultReporter 585 36.70 0.06 3.1% 0.0% + pre_solution_listener.ProgressListener 585 15.18 0.03 1.3% 0.0% + pre_solution_listener.BenchmarkTimer 585 3.41 0.01 0.3% 0.0% + pre_solution_listener.AllSolutionsIterator 585 1.11 0.00 0.1% 0.0% + pre_solution_listener.ReferenceValidator 585 0.99 0.00 0.1% 0.0% + cpu_reference_gemm_per_solution_inputs_changed 1 0.97 0.97 98.1% 0.0% + solve_cpu_slow_per_solution_inputs_changed 1 0.97 0.97 99.9% 0.0% + (untracked) 0.02 1.9% 0.0% + (untracked) 0.89 0.1% 0.0% + pre_solution_listener.SynchronizerValidator 585 0.10 0.00 0.0% 0.0% + pre_solution_listener.HardwareMonitorListener 585 0.01 0.00 0.0% 0.0% + post_solution 585 6.78 0.01 0.6% 0.0% + post_solution_log 585 5.47 0.01 80.7% 0.0% + post_solution_reporting 585 0.99 0.00 14.6% 0.0% + post_solution_perf_calc 585 0.20 0.00 2.9% 0.0% + (untracked) 0.07 1.1% 0.0% + post_solution_result_file 585 0.02 0.00 0.2% 0.0% + post_solution_validation 585 0.01 0.00 0.2% 0.0% + post_solution_sol_advance 585 0.01 0.00 0.2% 0.0% + post_solution_perf_reset 585 0.01 0.00 0.2% 0.0% + run_current_solution_check 585 1.61 0.00 0.1% 0.0% + post_problem 40 1.61 0.04 0.1% 0.0% + finalize_report 32 0.34 0.01 0.0% 0.0% + solution_selection 585 0.14 0.00 0.0% 0.0% + problem_report_index 40 0.13 0.00 0.0% 0.0% + warmup_runs 2 0.07 0.04 0.0% 0.0% + kernel_solving 1 0.06 0.06 0.0% 0.0% + problem_teardown 40 0.04 0.00 0.0% 0.0% + solution_teardown 585 0.02 0.00 0.0% 0.0% + warmup_events_setup 1 0.02 0.02 0.0% 0.0% + run_iteration_teardown 1 0.01 0.01 0.0% 0.0% + benchmark_runs 1 0.00 0.00 0.0% 0.0% + benchmark_pre_syncs 1 0.00 0.00 35.2% 0.0% + benchmark_post_syncs 1 0.00 0.00 35.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked C++ overhead) 11.94 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 5.33 0.0% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 32 4.73 0.15 88.7% 0.0% + timing_overhead 32 0.60 0.02 11.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 1.71 0.0% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 1 1.71 1.71 100.0% 0.0% + validate_warmups_listener.ReferenceValidator 1 1.18 1.18 69.3% 0.0% + validate_result_buffer_alloc 1 0.87 0.87 73.2% 0.0% + validate_gpu_readback 1 0.30 0.30 25.6% 0.0% + validate_element_comparison 1 0.01 0.01 0.8% 0.0% + validate_inputs_copy 1 0.00 0.00 0.1% 0.0% + validate_report 1 0.00 0.00 0.0% 0.0% + validate_setup 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.SynchronizerValidator 1 0.51 0.51 30.0% 0.0% + validate_synchronizer_check 1 0.51 0.51 100.0% 0.0% + validate_synchronizer_readback 1 0.34 0.34 66.6% 0.0% + validate_synchronizer_scan 1 0.17 0.17 33.3% 0.0% + validate_warmups_listener.BenchmarkTimer 1 0.00 0.00 0.3% 0.0% + validate_gpu_sync 1 0.00 0.00 87.9% 0.0% + validate_warmups_listener.MetaResultReporter 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 1 0.00 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 1 0.00 0.00 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 855 14,089.48 16.48 0.5% 0.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 855 13,420.73 15.70 95.3% 0.5% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 855 528.55 0.62 3.8% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_nocache 855 527.16 0.62 99.7% 0.0% + (untracked) 1.39 0.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 128.36 0.9% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 855 11.83 0.01 0.1% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + (untracked) 11,241.68 0.4% 0.4% + ------------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 855 8,355.06 9.77 0.3% 0.3% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_generate_solutions 32 4,866.91 152.09 0.2% 0.2% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_ga_between_evaluations 8 1,364.98 170.62 0.1% 0.1% + ------------------------------------------------------------------------------------------------------------------------- + python_write_cache 855 758.98 0.89 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_setup 8 29.94 3.74 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_clear_artifacts 24 17.63 0.73 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_read_scores 32 4.30 0.13 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_cache_check 831 0.58 0.00 0.0% 0.0% + ------------------------------------------------------------------------------------------------------------------------- + python_ductile_verification 8 0.30 0.04 0.0% 0.0% + + Overlapping Totals (wrap other timers; not summed into the tree) + --------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 32 14,691.68 459.12 0.6% + main_total 32 11,190.67 349.71 0.4% + problem_total 40 4,882.23 122.06 0.2% + solution_total 585 1,191.28 2.04 0.0% + run_iteration_total 1 3.64 3.64 0.0% + validate_total 1 1.18 1.18 0.0% + validate_check_results_typed 1 1.18 1.18 0.0% + +----------------------------------------------------------------------------------------------------------------------------- +HIERARCHY VALIDATION: OK (all child phases fit within their parents) + +TIME BREAKDOWN (visual, % of wall clock) +----------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 100.0% ################################################## + BUILD: solution generation & compile 98.1% ################################################# + Solution generation & build (Python) 97.3% ################################################ + python_kernel_compilation 85.1% ########################################## + python_kernel_build_src_co 38.2% ################### + python_kernel_build_src_co.compile 38.1% ################### + python_kernel_build_src_co.unbundle 0.1% + python_kernel_build_src_co.cache_check 0.0% + python_kernel_build_src_co.cache_populate 0.0% + python_kernel_build_src_co.setup 0.0% + python_kernel_build_src_co.move 0.0% + (untracked) 0.0% + python_kernel_codegen 23.2% ########### + python_kernel_write_assemble 9.6% #### + python_kernel_bench_setup 5.0% ## + python_kernel_bench_postprocess 4.0% # + python_benchpost_lib_construction 2.7% # + python_benchpost_library_write 0.9% + python_benchpost_naming 0.4% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_validate 3.1% # + python_kernel_build_co 0.8% + python_kernel_setup 0.8% + python_kernel_write_helpers 0.4% + (untracked) 0.1% + python_solution_generation 12.1% ###### + python_solgen_forked_solutions 12.1% ###### + python_solgen_custom_kernels 0.0% + python_solgen_fork_permutations 0.0% + (untracked) 0.0% + Ductile GA search (Python) 0.9% + python_ductile_initial_sampling 0.7% + python_ductile_generate_solutions 0.2% + python_ductile_ga_between_evaluations 0.1% + python_ductile_setup 0.0% + python_ductile_clear_artifacts 0.0% + python_ductile_read_scores 0.0% + python_ductile_verification 0.0% + SHARED: cache & solution files 0.9% + python_write_solutions 0.5% + python_wsol_dump 0.5% + python_wsol_prepare 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + python_solution_indexing 0.3% + python_write_cache 0.0% + python_cache_check 0.0% + RUN: benchmark client 0.5% + Client launch (startup, setup & teardown) 0.4% + hip_initialization 0.2% + process_exit 0.1% + main_problem_factory 0.0% + factory_hip_get_device_properties 0.0% + (untracked) 0.0% + main_teardown_locals 0.0% + process_startup 0.0% + data_init_setup 0.0% + code_object_loading 0.0% + library_loading 0.0% + lazy_loading_init 0.0% + main_parse_args 0.0% + reporter_setup 0.0% + python_client_write_run_script 0.0% + listener_setup 0.0% + solution_iterator_setup 0.0% + Input data preparation 0.2% + gpu_input_preparation 0.1% + gpu_input_prep_init_cpu_inputs 0.1% + gpu_input_prep_copy_valid_to_gpu 0.1% + gpu_input_prep_copy_swizzled 0.0% + gpu_input_prep_batched_inputs 0.0% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + (untracked) 0.0% + pre_solution_listener.DataInitialization 0.0% + pre_solution_mx_init_data 0.0% + pre_solution_mx_side_regenerate 0.0% + pre_solution_mx_side_cache_hit 0.0% + (untracked) 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 0.0% + pre_solution_mx_cpu_sync_no_regen 0.0% + pre_solution_mx_copy_valid_to_gpu_after_regen 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 0.0% + pre_solution_mx_cpu_sync_after_regen 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 0.0% + pre_solution_mx_check 0.0% + (untracked) 0.0% + cpu_data_init 0.0% + rotating_buffer_preparation 0.0% + gpu_input_reset 0.0% + gpu_input_reset_copy_valid_to_gpu 0.0% + gpu_input_reset_copy_swizzled 0.0% + gpu_input_reset_batched_inputs 0.0% + gpu_input_reset_copy_inputs_d2d 0.0% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_slow_path 0.0% + Per-solution bookkeeping & reporting 0.0% + pre_solution_listener.MetaResultReporter 0.0% + pre_solution_listener.ProgressListener 0.0% + post_solution_log 0.0% + pre_solution_listener.BenchmarkTimer 0.0% + run_current_solution_check 0.0% + post_problem 0.0% + pre_solution_listener.AllSolutionsIterator 0.0% + post_solution_reporting 0.0% + finalize_report 0.0% + post_solution_perf_calc 0.0% + solution_selection 0.0% + problem_report_index 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + problem_teardown 0.0% + solution_teardown 0.0% + post_solution_result_file 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + post_solution_validation 0.0% + post_solution_sol_advance 0.0% + post_solution_perf_reset 0.0% + run_iteration_teardown 0.0% + Reference GEMM (CPU) 0.0% + cpu_reference_gemm 0.0% + solve_cpu_slow 0.0% + pre_solution_listener.ReferenceValidator 0.0% + cpu_reference_gemm_per_solution_inputs_changed 0.0% + solve_cpu_slow_per_solution_inputs_changed 0.0% + (untracked) 0.0% + Validation 0.0% + validate_warmups_listener.ReferenceValidator 0.0% + validate_result_buffer_alloc 0.0% + validate_gpu_readback 0.0% + validate_element_comparison 0.0% + validate_inputs_copy 0.0% + validate_report 0.0% + validate_setup 0.0% + validate_warmups_listener.SynchronizerValidator 0.0% + validate_synchronizer_check 0.0% + validate_synchronizer_readback 0.0% + validate_synchronizer_scan 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.DataInitialization 0.0% + validate_warmups_listener.ProgressListener 0.0% + Kernel launch & execution 0.0% + warmup_runs 0.0% + kernel_solving 0.0% + warmup_events_setup 0.0% + validate_warmups_listener.BenchmarkTimer 0.0% + validate_gpu_sync 0.0% + benchmark_runs 0.0% + benchmark_pre_syncs 0.0% + benchmark_post_syncs 0.0% + MEASUREMENT OVERHEAD (instrumentation) 0.0% + main_timing_calibration 0.0% + flush_timing_buffer 0.0% + (timer overhead subtracted from parent timers) 0.0% + timing_overhead 0.0% + UNATTRIBUTED (untracked time & timer overhead) 0.4% + (untracked in python_benchmark_problems) 0.4% + (untracked in python_client_execution) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in post_solution) 0.0% + +KEY INSIGHTS (% of wall clock) +----------------------------------------------------------------------------------------------------------------------------- + Solution generation & build (Python) 2581.93s (97.3%) + Ductile GA search (Python) 23.70s (0.9%) + SHARED: cache & solution files 23.20s (0.9%) + Client launch (startup, setup & teardown) 9.45s (0.4%) + Input data preparation 4.80s (0.2%) + MEASUREMENT OVERHEAD (instrumentation) 0.36s (0.0%) + Per-solution bookkeeping & reporting 0.07s (0.0%) + Reference GEMM (CPU) 0.01s (0.0%) + Validation 0.00s (0.0%) + Kernel launch & execution 0.00s (0.0%) + UNATTRIBUTED (untracked time & timer overhead) 11.26s (0.4%) + + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +----------------------------------------------------------------------------------------------------------------------------- + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + --------------------------------------------------------------------------------------------------------------------------- + 2048 256 128 16 Float8 BFloat16 13.62 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_run/AGGREGATE_analysis.txt b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_run/AGGREGATE_analysis.txt new file mode 100644 index 000000000000..7039a2cf57fd --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/baseline_run/AGGREGATE_analysis.txt @@ -0,0 +1,753 @@ +================================================================================================================================= +TIMING ANALYSIS SUMMARY +================================================================================================================================= + +TIMING BY PURPOSE +--------------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +--------------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 2,401,081.82 100.0% + RUN: benchmark client 2,284,890.54 95.2% 95.2% + Reference GEMM (CPU) 1,044,373.53 45.7% 43.5% + cpu_reference_gemm 28,596 887,980.04 31.05 85.0% 37.0% + solve_cpu_slow 1,947 689,043.16 353.90 77.6% 28.7% + solve_cpu_fast 26,656 198,918.72 7.46 22.4% 8.3% + (untracked) 18.17 0.0% 0.0% + pre_solution_listener.ReferenceValidator 2,355,016 156,393.49 0.07 15.0% 6.5% + cpu_reference_gemm_per_solution_inputs_changed 1,146 156,323.60 136.41 100.0% 6.5% + solve_cpu_fast_per_solution_inputs_changed 1,127 144,255.07 128.00 92.3% 6.0% + solve_cpu_slow_per_solution_inputs_changed 19 12,067.95 635.16 7.7% 0.5% + (untracked) 0.58 0.0% 0.0% + pre_solution_ref_skipped 2,353,182 45.85 0.00 0.0% 0.0% + (untracked) 24.04 0.0% 0.0% + Input data preparation 500,723.39 21.9% 20.9% + gpu_input_reset 2,106,835 298,818.12 0.14 59.7% 12.4% + gpu_input_reset_copy_valid_to_gpu 719,180 175,364.38 0.24 58.7% 7.3% + gpu_input_reset_batched_inputs 1,473,329 92,844.41 0.06 31.1% 3.9% + gpu_input_reset_copy_inputs_d2d 1,473,329 24,642.09 0.02 8.2% 1.0% + gpu_input_reset_copy_swizzled 15,719 3,967.98 0.25 1.3% 0.2% + gpu_input_reset_fast_reset_output 633,454 1,776.88 0.00 0.6% 0.1% + gpu_input_reset_convert_to_inputs 1,473,329 131.80 0.00 0.0% 0.0% + gpu_input_reset_slow_path 1,473,329 28.33 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.02 0.01 0.0% 0.0% + gpu_input_reset_constant_inputs 4 0.01 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 4 0.00 0.00 0.0% 0.0% + (untracked) 62.21 0.0% 0.0% + cpu_data_init 28,596 104,977.26 3.67 21.0% 4.4% + pre_solution_listener.DataInitialization 2,355,016 54,893.95 0.02 11.0% 2.3% + pre_solution_mx_init_data 16,038 49,609.33 3.09 90.4% 2.1% + pre_solution_mx_side_regenerate 2,298 49,605.78 21.59 100.0% 2.1% + pre_solution_mx_side_cache_hit 29,778 2.59 0.00 0.0% 0.0% + (untracked) 0.96 0.0% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 14,889 2,988.73 0.20 5.4% 0.1% + pre_solution_mx_copy_valid_to_gpu_after_regen 1,149 1,142.31 0.99 2.1% 0.0% + pre_solution_mx_cpu_sync_after_regen 1,149 425.75 0.37 0.8% 0.0% + pre_solution_mx_cpu_sync_no_regen 14,889 313.25 0.02 0.6% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 14,889 312.41 0.02 0.6% 0.0% + pre_solution_mx_check 2,355,016 66.94 0.00 0.1% 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 1,149 23.67 0.02 0.0% 0.0% + (untracked) 11.55 0.0% 0.0% + gpu_input_preparation 28,641 41,335.21 1.44 8.3% 1.7% + gpu_input_prep_copy_valid_to_gpu 11,446 25,907.85 2.26 62.7% 1.1% + gpu_input_prep_copy_swizzled 1,151 10,866.81 9.44 26.3% 0.5% + gpu_input_prep_init_cpu_inputs 41 2,774.14 67.66 6.7% 0.1% + gpu_input_prep_batched_inputs 23,229 1,360.84 0.06 3.3% 0.1% + gpu_input_prep_copy_inputs_d2d 23,229 384.36 0.02 0.9% 0.0% + gpu_input_prep_fast_reset_output 5,403 22.99 0.00 0.1% 0.0% + gpu_input_prep_convert_to_inputs 23,229 8.08 0.00 0.0% 0.0% + gpu_input_prep_slow_path 23,229 2.12 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 43 0.49 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + (untracked) 7.49 0.0% 0.0% + rotating_buffer_preparation 28,641 698.85 0.02 0.1% 0.0% + Validation 329,115.09 14.4% 13.7% + validate_warmups_listener.SynchronizerValidator 2,106,827 165,415.38 0.08 50.3% 6.9% + validate_synchronizer_check 2,106,827 165,375.26 0.08 100.0% 6.9% + validate_synchronizer_scan 608,873 106,738.74 0.18 64.5% 4.4% + validate_synchronizer_readback 608,873 58,500.01 0.10 35.4% 2.4% + validate_synchronizer_inactive 1,497,978 31.08 0.00 0.0% 0.0% + (untracked) 105.43 0.1% 0.0% + (untracked) 40.12 0.0% 0.0% + validate_warmups_listener.ReferenceValidator 2,106,827 163,290.06 0.08 49.6% 6.8% + validate_gpu_readback 1,907,339 94,421.18 0.05 57.8% 3.9% + validate_element_comparison 1,907,339 59,441.20 0.03 36.4% 2.5% + validate_result_buffer_alloc 1,907,339 8,739.14 0.00 5.4% 0.4% + validate_inputs_copy 1,905,787 212.65 0.00 0.1% 0.0% + validate_setup 1,905,883 55.78 0.00 0.0% 0.0% + validate_report 1,907,339 52.77 0.00 0.0% 0.0% + (untracked) 367.35 0.2% 0.0% + validate_warmups_listener.MetaResultReporter 2,106,827 136.41 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 2,106,827 72.06 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 2,106,827 69.82 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 2,106,827 66.13 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 2,106,827 65.23 0.00 0.0% 0.0% + Client launch (startup, setup & teardown) 289,773.74 12.7% 12.1% + hip_initialization 855 125,416.91 146.69 43.3% 5.2% + process_exit 855 67,796.11 79.29 23.4% 2.8% + main_problem_factory 855 30,518.37 35.69 10.5% 1.3% + factory_hip_get_device_properties 192 30,325.24 157.94 99.4% 1.3% + (untracked) 193.13 0.6% 0.0% + data_init_setup 855 20,125.24 23.54 6.9% 0.8% + process_startup 855 12,583.38 14.72 4.3% 0.5% + main_teardown_locals 855 11,812.48 13.82 4.1% 0.5% + code_object_loading 855 10,438.69 12.21 3.6% 0.4% + library_loading 855 9,704.53 11.35 3.3% 0.4% + lazy_loading_init 855 894.36 1.05 0.3% 0.0% + main_parse_args 855 361.93 0.42 0.1% 0.0% + reporter_setup 855 59.03 0.07 0.0% 0.0% + python_client_write_run_script 855 40.28 0.05 0.0% 0.0% + listener_setup 855 13.25 0.02 0.0% 0.0% + solution_iterator_setup 855 9.16 0.01 0.0% 0.0% + Per-solution bookkeeping & reporting 54,750.84 2.4% 2.3% + post_solution_log 2,355,016 17,525.78 0.01 32.0% 0.7% + pre_solution_listener.BenchmarkTimer 2,355,016 11,577.12 0.00 21.1% 0.5% + run_current_solution_check 2,355,016 6,825.34 0.00 12.5% 0.3% + post_solution_reporting 2,355,016 5,414.85 0.00 9.9% 0.2% + pre_solution_listener.ProgressListener 2,355,016 2,976.28 0.00 5.4% 0.1% + pre_solution_listener.AllSolutionsIterator 2,355,016 2,862.81 0.00 5.2% 0.1% + post_problem 28,641 1,717.04 0.06 3.1% 0.1% + run_iteration_teardown 2,106,835 1,469.03 0.00 2.7% 0.1% + pre_solution_listener.MetaResultReporter 2,355,016 1,451.38 0.00 2.7% 0.1% + post_solution_perf_calc 2,355,016 620.73 0.00 1.1% 0.0% + solution_selection 2,355,016 597.01 0.00 1.1% 0.0% + post_solution_validation 2,355,016 589.90 0.00 1.1% 0.0% + finalize_report 855 476.73 0.56 0.9% 0.0% + pre_solution_listener.SynchronizerValidator 2,355,016 320.74 0.00 0.6% 0.0% + solution_teardown 2,355,016 80.13 0.00 0.1% 0.0% + post_solution_perf_reset 2,355,016 63.52 0.00 0.1% 0.0% + post_solution_result_file 2,355,016 54.04 0.00 0.1% 0.0% + pre_solution_listener.HardwareMonitorListener 2,355,016 53.95 0.00 0.1% 0.0% + post_solution_sol_advance 2,355,016 46.72 0.00 0.1% 0.0% + problem_report_index 28,641 23.70 0.00 0.0% 0.0% + problem_teardown 28,641 4.07 0.00 0.0% 0.0% + Kernel launch & execution 35,714.28 1.6% 1.5% + warmup_runs 4,213,654 21,843.85 0.01 61.2% 0.9% + validate_warmups_listener.BenchmarkTimer 2,106,827 7,041.82 0.00 19.7% 0.3% + validate_gpu_sync 2,106,827 6,985.70 0.00 99.2% 0.3% + (untracked) 56.12 0.8% 0.0% + kernel_solving 2,106,835 5,531.00 0.00 15.5% 0.2% + warmup_events_setup 2,106,835 1,009.36 0.00 2.8% 0.0% + benchmark_runs 2,106,835 288.25 0.00 0.8% 0.0% + benchmark_validate_enqueues 319 138.66 0.43 48.1% 0.0% + validate_enqueues_listener.BenchmarkTimer 319 136.50 0.43 98.4% 0.0% + benchmark_sleep 319 136.44 0.43 100.0% 0.0% + benchmark_event_elapsed_time 8 0.02 0.00 0.0% 0.0% + benchmark_event_destroy 8 0.01 0.00 0.0% 0.0% + (untracked) 0.03 0.0% 0.0% + validate_enqueues_listener.ProgressListener 319 1.89 0.01 1.4% 0.0% + validate_enqueues_listener.MetaResultReporter 319 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.HardwareMonitorListener 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.SynchronizerValidator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 319 0.01 0.00 0.0% 0.0% + (untracked) 0.20 0.1% 0.0% + benchmark_pre_syncs 2,106,835 57.56 0.00 20.0% 0.0% + benchmark_post_syncs 2,106,835 52.30 0.00 18.1% 0.0% + gpu_kernel_execution 319 39.61 0.12 13.7% 0.0% + benchmark_events_setup 319 0.09 0.00 0.0% 0.0% + benchmark_events_teardown 319 0.04 0.00 0.0% 0.0% + Library client (Python) 24,635.19 1.1% 1.0% + python_library_client_create_library 10 21,176.98 2,117.70 86.0% 0.9% + python_library_client_execution 10 3,324.12 332.41 13.5% 0.1% + python_library_client_subprocess 10 3,318.02 331.80 99.8% 0.1% + python_library_client_write_run_script 10 0.84 0.08 0.0% 0.0% + (untracked) 5.26 0.2% 0.0% + python_library_client_write_configs 10 118.05 11.80 0.5% 0.0% + python_library_client_setup 10 16.05 1.60 0.1% 0.0% + Library logic (Python) 5,804.47 0.3% 0.2% + python_logic_write 26 5,475.33 210.59 94.3% 0.2% + python_logic_parse_solutions 26 259.38 9.98 4.5% 0.0% + python_logic_analyze 26 69.77 2.68 1.2% 0.0% + BUILD: solution generation & compile 63,058.11 2.6% 2.6% + Solution generation & build (Python) 43,478.00 68.9% 1.8% + python_kernel_compilation 32 43,478.00 1,358.69 100.0% 1.8% + python_kernel_build_src_co 32 20,373.20 636.66 46.9% 0.8% + python_kernel_build_src_co.compile 4 20,291.46 5,072.87 99.6% 0.8% + python_kernel_build_src_co.cache_check 32 51.79 1.62 0.3% 0.0% + python_kernel_build_src_co.unbundle 4 22.95 5.74 0.1% 0.0% + python_kernel_build_src_co.cache_populate 4 3.07 0.77 0.0% 0.0% + python_kernel_build_src_co.setup 32 0.79 0.02 0.0% 0.0% + python_kernel_build_src_co.move 4 0.24 0.06 0.0% 0.0% + (untracked) 2.90 0.0% 0.0% + python_kernel_codegen 32 16,226.35 507.07 37.3% 0.7% + python_kernel_write_assemble 32 1,887.76 58.99 4.3% 0.1% + python_kernel_bench_setup 32 1,758.49 54.95 4.0% 0.1% + python_kernel_validate 32 1,519.26 47.48 3.5% 0.1% + python_kernel_build_co 32 623.12 19.47 1.4% 0.0% + python_kernel_bench_postprocess 32 454.53 14.20 1.0% 0.0% + python_benchpost_lib_construction 32 320.06 10.00 70.4% 0.0% + python_benchpost_library_write 32 78.93 2.47 17.4% 0.0% + python_benchpost_naming 32 50.83 1.59 11.2% 0.0% + python_benchpost_client_config 32 3.75 0.12 0.8% 0.0% + (untracked) 0.96 0.2% 0.0% + python_kernel_write_helpers 32 449.80 14.06 1.0% 0.0% + python_kernel_setup 32 135.43 4.23 0.3% 0.0% + (untracked) 50.05 0.1% 0.0% + Ductile GA search (Python) 19,580.11 31.1% 0.8% + python_ductile_initial_sampling 8 13,596.32 1,699.54 69.4% 0.6% + python_ductile_generate_solutions 32 4,572.39 142.89 23.4% 0.2% + python_ductile_ga_between_evaluations 8 1,355.61 169.45 6.9% 0.1% + python_ductile_setup 8 29.52 3.69 0.2% 0.0% + python_ductile_clear_artifacts 32 21.63 0.68 0.1% 0.0% + python_ductile_read_scores 32 4.31 0.13 0.0% 0.0% + python_ductile_verification 8 0.33 0.04 0.0% 0.0% + MEASUREMENT OVERHEAD (instrumentation) 52,754.57 2.2% 2.2% + flush_timing_buffer 865 33,553.73 38.79 63.6% 1.4% + main_timing_calibration 855 9,354.43 10.94 17.7% 0.4% + timing_overhead 855 5,093.55 5.96 9.7% 0.2% + (timer overhead subtracted from parent timers) 4,752.86 9.0% 0.2% + SHARED: cache & solution files 1,541.95 0.1% 0.1% + python_cache_check 831 1,249.54 1.50 81.0% 0.1% + python_write_solutions 855 198.78 0.23 12.9% 0.0% + python_wsol_dump 32 64.63 2.02 32.5% 0.0% + python_wsol_prepare 855 61.58 0.07 31.0% 0.0% + python_wsol_prepare_cache 823 58.61 0.07 95.2% 0.0% + python_wsol_prepare_nocache 32 0.96 0.03 1.6% 0.0% + (untracked) 2.00 3.2% 0.0% + python_wsol_header 855 60.44 0.07 30.4% 0.0% + (untracked) 12.13 6.1% 0.0% + python_write_cache 32 59.62 1.86 3.9% 0.0% + python_solution_indexing 32 34.01 1.06 2.2% 0.0% + UNATTRIBUTED (untracked time & timer overhead) -1,163.34 -0.0% -0.0% + (untracked in python_benchmark_problems) 2,280.60 0.0% 0.1% + (untracked in validate_warmups) 507.43 0.0% 0.0% + (untracked in pre_solution) 490.58 0.0% 0.0% + (untracked in pre_problem) 248.49 0.0% 0.0% + (untracked in python_library_logic) 9.08 0.0% 0.0% + (untracked in python_client_writer) 8.69 0.0% 0.0% + (children exceed benchmark_runs) -2.74 0.0% -0.0% + (children exceed post_solution) -148.90 0.0% -0.0% + (children exceed python_client_execution) -4,556.57 0.0% -0.2% + +TIMING BY TIMER NESTING (legacy view) +--------------------------------------------------------------------------------------------------------------------------------- + Category Count Total (ms) Mean (ms) % of Parent % of Wall +--------------------------------------------------------------------------------------------------------------------------------- + python_benchmark_problems 184 2,365,871.53 12,858.00 98.5% + ----------------------------------------------------------------------------------------------------------------------------- + python_client_execution 855 2,298,990.88 2,688.88 97.2% 95.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Data Preparation 1,334,057.97 58.0% 55.6% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_problem 28,641 993,205.79 34.68 74.4% 41.4% + cpu_reference_gemm 28,596 887,980.04 31.05 89.4% 37.0% + solve_cpu_slow 1,947 689,043.16 353.90 77.6% 28.7% + solve_cpu_fast 26,656 198,918.72 7.46 22.4% 8.3% + (untracked) 18.17 0.0% 0.0% + cpu_data_init 28,596 104,977.26 3.67 10.6% 4.4% + (untracked) 248.49 0.0% 0.0% + gpu_input_reset 2,106,835 298,818.12 0.14 22.4% 12.4% + gpu_input_reset_copy_valid_to_gpu 719,180 175,364.38 0.24 58.7% 7.3% + gpu_input_reset_batched_inputs 1,473,329 92,844.41 0.06 31.1% 3.9% + gpu_input_reset_copy_inputs_d2d 1,473,329 24,642.09 0.02 8.2% 1.0% + gpu_input_reset_copy_swizzled 15,719 3,967.98 0.25 1.3% 0.2% + gpu_input_reset_fast_reset_output 633,454 1,776.88 0.00 0.6% 0.1% + gpu_input_reset_convert_to_inputs 1,473,329 131.80 0.00 0.0% 0.0% + (untracked) 62.21 0.0% 0.0% + gpu_input_reset_slow_path 1,473,329 28.33 0.00 0.0% 0.0% + gpu_input_reset_rotating_init 2 0.02 0.01 0.0% 0.0% + gpu_input_reset_constant_inputs 4 0.01 0.00 0.0% 0.0% + gpu_input_reset_init_cpu_inputs 4 0.00 0.00 0.0% 0.0% + gpu_input_preparation 28,641 41,335.21 1.44 3.1% 1.7% + gpu_input_prep_copy_valid_to_gpu 11,446 25,907.85 2.26 62.7% 1.1% + gpu_input_prep_copy_swizzled 1,151 10,866.81 9.44 26.3% 0.5% + gpu_input_prep_init_cpu_inputs 41 2,774.14 67.66 6.7% 0.1% + gpu_input_prep_batched_inputs 23,229 1,360.84 0.06 3.3% 0.1% + gpu_input_prep_copy_inputs_d2d 23,229 384.36 0.02 0.9% 0.0% + gpu_input_prep_fast_reset_output 5,403 22.99 0.00 0.1% 0.0% + gpu_input_prep_convert_to_inputs 23,229 8.08 0.00 0.0% 0.0% + (untracked) 7.49 0.0% 0.0% + gpu_input_prep_slow_path 23,229 2.12 0.00 0.0% 0.0% + gpu_input_prep_constant_inputs 43 0.49 0.01 0.0% 0.0% + gpu_input_prep_rotating_init 1 0.01 0.01 0.0% 0.0% + rotating_buffer_preparation 28,641 698.85 0.02 0.1% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Reference Computation 336,664.34 14.6% 14.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + validate_warmups 2,106,827 336,664.34 0.16 100.0% 14.0% + validate_warmups_listener.SynchronizerValidator 2,106,827 165,415.38 0.08 49.1% 6.9% + validate_synchronizer_check 2,106,827 165,375.26 0.08 100.0% 6.9% + validate_synchronizer_scan 608,873 106,738.74 0.18 64.5% 4.4% + validate_synchronizer_readback 608,873 58,500.01 0.10 35.4% 2.4% + (untracked) 105.43 0.1% 0.0% + validate_synchronizer_inactive 1,497,978 31.08 0.00 0.0% 0.0% + (untracked) 40.12 0.0% 0.0% + validate_warmups_listener.ReferenceValidator 2,106,827 163,290.06 0.08 48.5% 6.8% + validate_gpu_readback 1,907,339 94,421.18 0.05 57.8% 3.9% + validate_element_comparison 1,907,339 59,441.20 0.03 36.4% 2.5% + validate_result_buffer_alloc 1,907,339 8,739.14 0.00 5.4% 0.4% + (untracked) 367.35 0.2% 0.0% + validate_inputs_copy 1,905,787 212.65 0.00 0.1% 0.0% + validate_setup 1,905,883 55.78 0.00 0.0% 0.0% + validate_report 1,907,339 52.77 0.00 0.0% 0.0% + validate_warmups_listener.BenchmarkTimer 2,106,827 7,041.82 0.00 2.1% 0.3% + validate_gpu_sync 2,106,827 6,985.70 0.00 99.2% 0.3% + (untracked) 56.12 0.8% 0.0% + (untracked) 507.43 0.2% 0.0% + validate_warmups_listener.MetaResultReporter 2,106,827 136.41 0.00 0.0% 0.0% + validate_warmups_listener.AllSolutionsIterator 2,106,827 72.06 0.00 0.0% 0.0% + validate_warmups_listener.DataInitialization 2,106,827 69.82 0.00 0.0% 0.0% + validate_warmups_listener.ProgressListener 2,106,827 66.13 0.00 0.0% 0.0% + validate_warmups_listener.HardwareMonitorListener 2,106,827 65.23 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + C++ Client Setup & Teardown 299,128.16 13.0% 12.5% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + hip_initialization 855 125,416.91 146.69 41.9% 5.2% + process_exit 855 67,796.11 79.29 22.7% 2.8% + main_problem_factory 855 30,518.37 35.69 10.2% 1.3% + factory_hip_get_device_properties 192 30,325.24 157.94 99.4% 1.3% + (untracked) 193.13 0.6% 0.0% + data_init_setup 855 20,125.24 23.54 6.7% 0.8% + process_startup 855 12,583.38 14.72 4.2% 0.5% + main_teardown_locals 855 11,812.48 13.82 3.9% 0.5% + code_object_loading 855 10,438.69 12.21 3.5% 0.4% + library_loading 855 9,704.53 11.35 3.2% 0.4% + main_timing_calibration 855 9,354.43 10.94 3.1% 0.4% + lazy_loading_init 855 894.36 1.05 0.3% 0.0% + main_parse_args 855 361.93 0.42 0.1% 0.0% + reporter_setup 855 59.03 0.07 0.0% 0.0% + python_client_write_run_script 855 40.28 0.05 0.0% 0.0% + listener_setup 855 13.25 0.02 0.0% 0.0% + solution_iterator_setup 855 9.16 0.01 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Kernel Execution 295,049.68 12.8% 12.3% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + pre_solution 2,355,016 231,020.29 0.10 78.3% 9.6% + pre_solution_listener.ReferenceValidator 2,355,016 156,393.49 0.07 67.7% 6.5% + cpu_reference_gemm_per_solution_inputs_changed 1,146 156,323.60 136.41 100.0% 6.5% + solve_cpu_fast_per_solution_inputs_changed 1,127 144,255.07 128.00 92.3% 6.0% + solve_cpu_slow_per_solution_inputs_changed 19 12,067.95 635.16 7.7% 0.5% + (untracked) 0.58 0.0% 0.0% + pre_solution_ref_skipped 2,353,182 45.85 0.00 0.0% 0.0% + (untracked) 24.04 0.0% 0.0% + pre_solution_listener.DataInitialization 2,355,016 54,893.95 0.02 23.8% 2.3% + pre_solution_mx_init_data 16,038 49,609.33 3.09 90.4% 2.1% + pre_solution_mx_side_regenerate 2,298 49,605.78 21.59 100.0% 2.1% + pre_solution_mx_side_cache_hit 29,778 2.59 0.00 0.0% 0.0% + (untracked) 0.96 0.0% 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 14,889 2,988.73 0.20 5.4% 0.1% + pre_solution_mx_copy_valid_to_gpu_after_regen 1,149 1,142.31 0.99 2.1% 0.0% + pre_solution_mx_cpu_sync_after_regen 1,149 425.75 0.37 0.8% 0.0% + pre_solution_mx_cpu_sync_no_regen 14,889 313.25 0.02 0.6% 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 14,889 312.41 0.02 0.6% 0.0% + pre_solution_mx_check 2,355,016 66.94 0.00 0.1% 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 1,149 23.67 0.02 0.0% 0.0% + (untracked) 11.55 0.0% 0.0% + pre_solution_listener.BenchmarkTimer 2,355,016 11,577.12 0.00 5.0% 0.5% + pre_solution_listener.ProgressListener 2,355,016 2,976.28 0.00 1.3% 0.1% + pre_solution_listener.AllSolutionsIterator 2,355,016 2,862.81 0.00 1.2% 0.1% + pre_solution_listener.MetaResultReporter 2,355,016 1,451.38 0.00 0.6% 0.1% + (untracked) 490.58 0.2% 0.0% + pre_solution_listener.SynchronizerValidator 2,355,016 320.74 0.00 0.1% 0.0% + pre_solution_listener.HardwareMonitorListener 2,355,016 53.95 0.00 0.0% 0.0% + post_solution 2,355,016 24,166.63 0.01 8.2% 1.0% + post_solution_log 2,355,016 17,525.78 0.01 72.5% 0.7% + post_solution_reporting 2,355,016 5,414.85 0.00 22.4% 0.2% + post_solution_perf_calc 2,355,016 620.73 0.00 2.6% 0.0% + post_solution_validation 2,355,016 589.90 0.00 2.4% 0.0% + post_solution_perf_reset 2,355,016 63.52 0.00 0.3% 0.0% + post_solution_result_file 2,355,016 54.04 0.00 0.2% 0.0% + post_solution_sol_advance 2,355,016 46.72 0.00 0.2% 0.0% + warmup_runs 4,213,654 21,843.85 0.01 7.4% 0.9% + run_current_solution_check 2,355,016 6,825.34 0.00 2.3% 0.3% + kernel_solving 2,106,835 5,531.00 0.00 1.9% 0.2% + post_problem 28,641 1,717.04 0.06 0.6% 0.1% + run_iteration_teardown 2,106,835 1,469.03 0.00 0.5% 0.1% + warmup_events_setup 2,106,835 1,009.36 0.00 0.3% 0.0% + solution_selection 2,355,016 597.01 0.00 0.2% 0.0% + finalize_report 855 476.73 0.56 0.2% 0.0% + benchmark_runs 2,106,835 285.51 0.00 0.1% 0.0% + benchmark_validate_enqueues 319 138.66 0.43 48.6% 0.0% + validate_enqueues_listener.BenchmarkTimer 319 136.50 0.43 98.4% 0.0% + benchmark_sleep 319 136.44 0.43 100.0% 0.0% + (untracked) 0.03 0.0% 0.0% + benchmark_event_elapsed_time 8 0.02 0.00 0.0% 0.0% + benchmark_event_destroy 8 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ProgressListener 319 1.89 0.01 1.4% 0.0% + (untracked) 0.20 0.1% 0.0% + validate_enqueues_listener.MetaResultReporter 319 0.02 0.00 0.0% 0.0% + validate_enqueues_listener.HardwareMonitorListener 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.SynchronizerValidator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.AllSolutionsIterator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.ReferenceValidator 319 0.01 0.00 0.0% 0.0% + validate_enqueues_listener.DataInitialization 319 0.01 0.00 0.0% 0.0% + benchmark_pre_syncs 2,106,835 57.56 0.00 20.2% 0.0% + benchmark_post_syncs 2,106,835 52.30 0.00 18.3% 0.0% + gpu_kernel_execution 319 39.61 0.12 13.9% 0.0% + benchmark_events_setup 319 0.09 0.00 0.0% 0.0% + benchmark_events_teardown 319 0.04 0.00 0.0% 0.0% + solution_teardown 2,355,016 80.13 0.00 0.0% 0.0% + problem_report_index 28,641 23.70 0.00 0.0% 0.0% + problem_teardown 28,641 4.07 0.00 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Timing Overhead 38,647.28 1.7% 1.6% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + flush_timing_buffer 865 33,553.73 38.79 86.8% 1.4% + timing_overhead 855 5,093.55 5.96 13.2% 0.2% + ----------------------------------------------------------------------------------------------------------------------------- + python_kernel_compilation 32 43,478.00 1,358.69 1.8% 1.8% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_src_co 32 20,373.20 636.66 46.9% 0.8% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_kernel_build_src_co.compile 4 20,291.46 5,072.87 99.6% 0.8% + python_kernel_build_src_co.cache_check 32 51.79 1.62 0.3% 0.0% + python_kernel_build_src_co.unbundle 4 22.95 5.74 0.1% 0.0% + python_kernel_build_src_co.cache_populate 4 3.07 0.77 0.0% 0.0% + (untracked) 2.90 0.0% 0.0% + python_kernel_build_src_co.setup 32 0.79 0.02 0.0% 0.0% + python_kernel_build_src_co.move 4 0.24 0.06 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_codegen 32 16,226.35 507.07 37.3% 0.7% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_assemble 32 1,887.76 58.99 4.3% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_setup 32 1,758.49 54.95 4.0% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_validate 32 1,519.26 47.48 3.5% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_build_co 32 623.12 19.47 1.4% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_bench_postprocess 32 454.53 14.20 1.0% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_benchpost_lib_construction 32 320.06 10.00 70.4% 0.0% + python_benchpost_library_write 32 78.93 2.47 17.4% 0.0% + python_benchpost_naming 32 50.83 1.59 11.2% 0.0% + python_benchpost_client_config 32 3.75 0.12 0.8% 0.0% + (untracked) 0.96 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_write_helpers 32 449.80 14.06 1.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_kernel_setup 32 135.43 4.23 0.3% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 50.05 0.1% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_initial_sampling 8 13,596.32 1,699.54 0.6% 0.6% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_generate_solutions 32 4,572.39 142.89 0.2% 0.2% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 2,280.60 0.1% 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_ga_between_evaluations 8 1,355.61 169.45 0.1% 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_cache_check 831 1,249.54 1.50 0.1% 0.1% + ----------------------------------------------------------------------------------------------------------------------------- + python_write_solutions 855 198.78 0.23 0.0% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_dump 32 64.63 2.02 32.5% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_prepare 855 61.58 0.07 31.0% 0.0% + . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + python_wsol_prepare_cache 823 58.61 0.07 95.2% 0.0% + (untracked) 2.00 3.2% 0.0% + python_wsol_prepare_nocache 32 0.96 0.03 1.6% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_wsol_header 855 60.44 0.07 30.4% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 12.13 6.1% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_write_cache 32 59.62 1.86 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_solution_indexing 32 34.01 1.06 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_setup 8 29.52 3.69 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_clear_artifacts 32 21.63 0.68 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_read_scores 32 4.31 0.13 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_ductile_verification 8 0.33 0.04 0.0% 0.0% + =============================================================================================================================== + python_client_writer 10 24,643.88 2,464.39 1.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_create_library 10 21,176.98 2,117.70 85.9% 0.9% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_execution 10 3,324.12 332.41 13.5% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_subprocess 10 3,318.02 331.80 99.8% 0.1% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + (untracked) 5.26 0.2% 0.0% + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + python_library_client_write_run_script 10 0.84 0.08 0.0% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_write_configs 10 118.05 11.80 0.5% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_library_client_setup 10 16.05 1.60 0.1% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 8.69 0.0% 0.0% + =============================================================================================================================== + python_library_logic 26 5,813.56 223.60 0.2% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_write 26 5,475.33 210.59 94.2% 0.2% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_parse_solutions 26 259.38 9.98 4.5% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + python_logic_analyze 26 69.77 2.68 1.2% 0.0% + ----------------------------------------------------------------------------------------------------------------------------- + (untracked) 9.08 0.2% 0.0% + + Overlapping Totals (wrap other timers; not summed into the tree) + ------------------------------------------------------------------------------------------------------------------------------- + python_client_subprocess 855 2,303,413.47 2,694.05 95.9% + main_total 855 2,177,641.12 2,546.95 90.7% + problem_total 28,641 1,970,235.51 68.79 82.1% + solution_total 2,355,016 933,104.61 0.40 38.9% + run_iteration_total 2,106,835 668,286.95 0.32 27.8% + validate_total 1,905,883 163,188.57 0.09 6.8% + validate_check_results_typed 1,907,339 162,915.98 0.09 6.8% + +--------------------------------------------------------------------------------------------------------------------------------- +HIERARCHY VALIDATION WARNINGS +--------------------------------------------------------------------------------------------------------------------------------- + WARNING: Child phases of python_client_execution (2,307,117.88 ms) exceed parent (2,303,743.49 ms) by 3,374.39 ms (0.1%) + +TIME BREAKDOWN (visual, % of wall clock) +--------------------------------------------------------------------------------------------------------------------------------- + TOTAL RUNTIME 100.0% ################################################## + RUN: benchmark client 95.2% ############################################### + Reference GEMM (CPU) 43.5% ##################### + cpu_reference_gemm 37.0% ################## + solve_cpu_slow 28.7% ############## + solve_cpu_fast 8.3% #### + (untracked) 0.0% + pre_solution_listener.ReferenceValidator 6.5% ### + cpu_reference_gemm_per_solution_inputs_changed 6.5% ### + solve_cpu_fast_per_solution_inputs_changed 6.0% ### + solve_cpu_slow_per_solution_inputs_changed 0.5% + (untracked) 0.0% + pre_solution_ref_skipped 0.0% + (untracked) 0.0% + Input data preparation 20.9% ########## + gpu_input_reset 12.4% ###### + gpu_input_reset_copy_valid_to_gpu 7.3% ### + gpu_input_reset_batched_inputs 3.9% # + gpu_input_reset_copy_inputs_d2d 1.0% + gpu_input_reset_copy_swizzled 0.2% + gpu_input_reset_fast_reset_output 0.1% + gpu_input_reset_convert_to_inputs 0.0% + gpu_input_reset_slow_path 0.0% + gpu_input_reset_rotating_init 0.0% + gpu_input_reset_constant_inputs 0.0% + gpu_input_reset_init_cpu_inputs 0.0% + (untracked) 0.0% + cpu_data_init 4.4% ## + pre_solution_listener.DataInitialization 2.3% # + pre_solution_mx_init_data 2.1% # + pre_solution_mx_side_regenerate 2.1% # + pre_solution_mx_side_cache_hit 0.0% + (untracked) 0.0% + pre_solution_mx_copy_valid_to_gpu_no_regen 0.1% + pre_solution_mx_copy_valid_to_gpu_after_regen 0.0% + pre_solution_mx_cpu_sync_after_regen 0.0% + pre_solution_mx_cpu_sync_no_regen 0.0% + pre_solution_mx_copy_inputs_d2d_no_regen 0.0% + pre_solution_mx_check 0.0% + pre_solution_mx_copy_inputs_d2d_after_regen 0.0% + (untracked) 0.0% + gpu_input_preparation 1.7% + gpu_input_prep_copy_valid_to_gpu 1.1% + gpu_input_prep_copy_swizzled 0.5% + gpu_input_prep_init_cpu_inputs 0.1% + gpu_input_prep_batched_inputs 0.1% + gpu_input_prep_copy_inputs_d2d 0.0% + gpu_input_prep_fast_reset_output 0.0% + gpu_input_prep_convert_to_inputs 0.0% + gpu_input_prep_slow_path 0.0% + gpu_input_prep_constant_inputs 0.0% + gpu_input_prep_rotating_init 0.0% + (untracked) 0.0% + rotating_buffer_preparation 0.0% + Validation 13.7% ###### + validate_warmups_listener.SynchronizerValidator 6.9% ### + validate_synchronizer_check 6.9% ### + validate_synchronizer_scan 4.4% ## + validate_synchronizer_readback 2.4% # + validate_synchronizer_inactive 0.0% + (untracked) 0.0% + (untracked) 0.0% + validate_warmups_listener.ReferenceValidator 6.8% ### + validate_gpu_readback 3.9% # + validate_element_comparison 2.5% # + validate_result_buffer_alloc 0.4% + validate_inputs_copy 0.0% + validate_setup 0.0% + validate_report 0.0% + (untracked) 0.0% + validate_warmups_listener.MetaResultReporter 0.0% + validate_warmups_listener.AllSolutionsIterator 0.0% + validate_warmups_listener.DataInitialization 0.0% + validate_warmups_listener.ProgressListener 0.0% + validate_warmups_listener.HardwareMonitorListener 0.0% + Client launch (startup, setup & teardown) 12.1% ###### + hip_initialization 5.2% ## + process_exit 2.8% # + main_problem_factory 1.3% + factory_hip_get_device_properties 1.3% + (untracked) 0.0% + data_init_setup 0.8% + process_startup 0.5% + main_teardown_locals 0.5% + code_object_loading 0.4% + library_loading 0.4% + lazy_loading_init 0.0% + main_parse_args 0.0% + reporter_setup 0.0% + python_client_write_run_script 0.0% + listener_setup 0.0% + solution_iterator_setup 0.0% + Per-solution bookkeeping & reporting 2.3% # + post_solution_log 0.7% + pre_solution_listener.BenchmarkTimer 0.5% + run_current_solution_check 0.3% + post_solution_reporting 0.2% + pre_solution_listener.ProgressListener 0.1% + pre_solution_listener.AllSolutionsIterator 0.1% + post_problem 0.1% + run_iteration_teardown 0.1% + pre_solution_listener.MetaResultReporter 0.1% + post_solution_perf_calc 0.0% + solution_selection 0.0% + post_solution_validation 0.0% + finalize_report 0.0% + pre_solution_listener.SynchronizerValidator 0.0% + solution_teardown 0.0% + post_solution_perf_reset 0.0% + post_solution_result_file 0.0% + pre_solution_listener.HardwareMonitorListener 0.0% + post_solution_sol_advance 0.0% + problem_report_index 0.0% + problem_teardown 0.0% + Kernel launch & execution 1.5% + warmup_runs 0.9% + validate_warmups_listener.BenchmarkTimer 0.3% + validate_gpu_sync 0.3% + (untracked) 0.0% + kernel_solving 0.2% + warmup_events_setup 0.0% + benchmark_runs 0.0% + benchmark_validate_enqueues 0.0% + validate_enqueues_listener.BenchmarkTimer 0.0% + benchmark_sleep 0.0% + benchmark_event_elapsed_time 0.0% + benchmark_event_destroy 0.0% + (untracked) 0.0% + validate_enqueues_listener.ProgressListener 0.0% + validate_enqueues_listener.MetaResultReporter 0.0% + validate_enqueues_listener.HardwareMonitorListener 0.0% + validate_enqueues_listener.SynchronizerValidator 0.0% + validate_enqueues_listener.AllSolutionsIterator 0.0% + validate_enqueues_listener.ReferenceValidator 0.0% + validate_enqueues_listener.DataInitialization 0.0% + (untracked) 0.0% + benchmark_pre_syncs 0.0% + benchmark_post_syncs 0.0% + gpu_kernel_execution 0.0% + benchmark_events_setup 0.0% + benchmark_events_teardown 0.0% + Library client (Python) 1.0% + python_library_client_create_library 0.9% + python_library_client_execution 0.1% + python_library_client_subprocess 0.1% + python_library_client_write_run_script 0.0% + (untracked) 0.0% + python_library_client_write_configs 0.0% + python_library_client_setup 0.0% + Library logic (Python) 0.2% + python_logic_write 0.2% + python_logic_parse_solutions 0.0% + python_logic_analyze 0.0% + BUILD: solution generation & compile 2.6% # + Solution generation & build (Python) 1.8% + python_kernel_compilation 1.8% + python_kernel_build_src_co 0.8% + python_kernel_build_src_co.compile 0.8% + python_kernel_build_src_co.cache_check 0.0% + python_kernel_build_src_co.unbundle 0.0% + python_kernel_build_src_co.cache_populate 0.0% + python_kernel_build_src_co.setup 0.0% + python_kernel_build_src_co.move 0.0% + (untracked) 0.0% + python_kernel_codegen 0.7% + python_kernel_write_assemble 0.1% + python_kernel_bench_setup 0.1% + python_kernel_validate 0.1% + python_kernel_build_co 0.0% + python_kernel_bench_postprocess 0.0% + python_benchpost_lib_construction 0.0% + python_benchpost_library_write 0.0% + python_benchpost_naming 0.0% + python_benchpost_client_config 0.0% + (untracked) 0.0% + python_kernel_write_helpers 0.0% + python_kernel_setup 0.0% + (untracked) 0.0% + Ductile GA search (Python) 0.8% + python_ductile_initial_sampling 0.6% + python_ductile_generate_solutions 0.2% + python_ductile_ga_between_evaluations 0.1% + python_ductile_setup 0.0% + python_ductile_clear_artifacts 0.0% + python_ductile_read_scores 0.0% + python_ductile_verification 0.0% + MEASUREMENT OVERHEAD (instrumentation) 2.2% # + flush_timing_buffer 1.4% + main_timing_calibration 0.4% + timing_overhead 0.2% + (timer overhead subtracted from parent timers) 0.2% + SHARED: cache & solution files 0.1% + python_cache_check 0.1% + python_write_solutions 0.0% + python_wsol_dump 0.0% + python_wsol_prepare 0.0% + python_wsol_prepare_cache 0.0% + python_wsol_prepare_nocache 0.0% + (untracked) 0.0% + python_wsol_header 0.0% + (untracked) 0.0% + python_write_cache 0.0% + python_solution_indexing 0.0% + UNATTRIBUTED (untracked time & timer overhead) -0.0% + (untracked in python_benchmark_problems) 0.1% + (untracked in validate_warmups) 0.0% + (untracked in pre_solution) 0.0% + (untracked in pre_problem) 0.0% + (untracked in python_library_logic) 0.0% + (untracked in python_client_writer) 0.0% + (children exceed benchmark_runs) -0.0% + (children exceed post_solution) -0.0% + (children exceed python_client_execution) -0.2% + +KEY INSIGHTS (% of wall clock) +--------------------------------------------------------------------------------------------------------------------------------- + Reference GEMM (CPU) 1044.37s (43.5%) + Input data preparation 500.72s (20.9%) + Validation 329.12s (13.7%) + Client launch (startup, setup & teardown) 289.77s (12.1%) + Per-solution bookkeeping & reporting 54.75s (2.3%) + MEASUREMENT OVERHEAD (instrumentation) 48.00s (2.0%) + Solution generation & build (Python) 43.48s (1.8%) + Kernel launch & execution 35.71s (1.5%) + Library client (Python) 24.64s (1.0%) + Ductile GA search (Python) 19.58s (0.8%) + Library logic (Python) 5.80s (0.2%) + SHARED: cache & solution files 1.54s (0.1%) + UNATTRIBUTED (untracked time & timer overhead) -1.16s (-0.0%) + + CPU reference / GPU execution ratio: 22420.9x + >>> CPU reference > 10x GPU execution time! + +TOP 10 SLOWEST PROBLEMS (by CPU reference time) +--------------------------------------------------------------------------------------------------------------------------------- + M N K Batch TypeA TypeD CPU Ref (ms) GPU (ms) + ------------------------------------------------------------------------------------------------------------------------------- + 4111 4111 4111 1 Float8 Float8 132701.21 0.00 + 4111 4111 4111 1 Float8 Float8 111415.30 0.00 + 4111 4111 4111 1 Int8 Int8 84486.58 0.00 + 4111 4111 4111 1 Int8 BFloat16 79404.49 0.00 + 4111 4111 4111 1 Int8 Int32 77682.79 0.00 + 4111 4111 4111 1 Int8 Half 55478.46 0.00 + 43776 10624 6400 1 Float8 BFloat16 15800.60 0.00 + 71936 4096 7168 1 Float8 BFloat16 13588.45 0.00 + 13056 7680 21504 1 Float8 BFloat16 13101.60 0.00 + 13696 15488 8448 1 Float8 BFloat16 9323.37 0.00 + diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/compare_aggregates.py b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/compare_aggregates.py new file mode 100755 index 000000000000..fb9c1a86826e --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/compare_aggregates.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Compare two per-config timing passes side by side. + +Each argument is the OUT directory of a run_per_config.sh or run_per_config_build.sh pass: it holds +AGGREGATE_analysis.txt (written by scripts/aggregate_timing.py) and one /times.txt per +config. The script prints the "TIMING BY PURPOSE" table of both aggregates side by side with the +change in percent, then the summed pytest wall time, the slowest configs and every config that +got more than 2 s slower. + +Usage: compare_aggregates.py BEFORE_DIR AFTER_DIR [--max-depth N] +""" + +import argparse +import glob +import os +import re + + +def purpose_rows(path): + """(indent, category, milliseconds) for each row of the TIMING BY PURPOSE table.""" + rows = [] + inside = False + with open(path) as f: + for line in f: + if line.startswith("TIMING BY PURPOSE"): + inside = True + continue + if inside and line.startswith("TIMING BY TIMER NESTING"): + break + m = re.match(r"^(\s+)(\S.*?)\s{2,}(?:([\d,]+)\s+)?([\d,]+\.\d\d)\s", line) + if inside and m: + rows.append((len(m.group(1)), m.group(2).strip(), float(m.group(4).replace(",", "")))) + return rows + + +def keyed(rows): + """Rows keyed by their path in the category tree, so equal names at different places differ.""" + out = {} + stack = [] + for indent, name, ms in rows: + while stack and stack[-1][0] >= indent: + stack.pop() + stack.append((indent, name)) + out["/".join(n for _, n in stack)] = (indent, ms) + return out + + +def pytest_seconds(directory): + """Pytest wall seconds per config, from /times.txt.""" + seconds = {} + for path in glob.glob(os.path.join(directory, "*", "times.txt")): + with open(path) as f: + m = re.search(r"seconds=(\d+)", f.read()) + if m: + seconds[os.path.basename(os.path.dirname(path))] = int(m.group(1)) + return seconds + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("before", help="OUT directory of the 'before' pass") + parser.add_argument("after", help="OUT directory of the 'after' pass") + parser.add_argument("--max-depth", type=int, default=8, help="deepest category level printed") + args = parser.parse_args() + + a = keyed(purpose_rows(os.path.join(args.before, "AGGREGATE_analysis.txt"))) + b = keyed(purpose_rows(os.path.join(args.after, "AGGREGATE_analysis.txt"))) + print(f"{'Category':70s} {'before s':>10s} {'after s':>10s} {'delta':>8s}") + for key in list(a) + [k for k in b if k not in a]: + indent = (a.get(key) or b.get(key))[0] + if (indent - 4) // 2 > args.max_depth: + continue + x = a.get(key, (0, 0))[1] / 1000 + y = b.get(key, (0, 0))[1] / 1000 + if max(x, y) < 1: + continue + delta = f"{(y - x) / x * 100:+.1f}%" if x else "new" + width = 70 - indent + 4 + print(f"{' ' * (indent - 4)}{key.split('/')[-1][:width]:{width}s} {x:10.1f} {y:10.1f} {delta:>8s}") + + tb = pytest_seconds(args.before) + ta = pytest_seconds(args.after) + total_b = sum(tb.values()) + total_a = sum(ta.values()) + change = f" ({(total_a - total_b) / total_b * 100:+.1f}%)" if total_b else "" + print(f"\npytest wall sum: before {total_b} s, after {total_a} s{change}") + print("largest per-config (before -> after s):") + for k in sorted(tb, key=lambda k: -tb[k])[:12]: + print(f" {k:60s} {tb[k]:5d} -> {ta.get(k, 0):5d}") + slower = sorted(((k, tb[k], ta[k]) for k in tb if k in ta and ta[k] > tb[k] + 2), key=lambda t: t[1] - t[2]) + print("slower by more than 2 s:", slower if slower else "none") + + +if __name__ == "__main__": + main() diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/run_side.sh b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/run_side.sh new file mode 100755 index 000000000000..d0b2ab32b314 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/before_after/run_side.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# One side ("before" or "after") of a before/after timing of the common tests. Start one instance +# per side, at the same time, each with its own source tree, client, cores and GPU, so that both +# sides see the same host load. Each side runs: +# 1. split/run_split.sh: the CI-like build phase then run phase (uninstrumented, -n 4), and +# records the configs whose run passed (/split_run_passed.txt); +# 2. waits until every side in SIDES has finished step 1, then writes the shared config list +# (/configs.txt): the union of the configs that passed in any side's split run phase; +# 3. per_config/run_per_config_build.sh over that list, instrumented, caches cleared once before +# the first config, artifacts kept; +# 4. per_config/run_per_config.sh on those artifacts, instrumented, caches not cleared. +# Compare the two sides with before_after/compare_aggregates.py. +# Usage: bash run_side.sh ( is one of SIDES) +# Env: ROOT work directory shared by the sides (required); 's output goes to +# $ROOT// +# TL, CLIENT this side's TensileLite tree and prebuilt tensilelite-client (required) +# SIDES side names, default "before after" +# GPU_TARGET, CORES, GPU, ANALYZE_DIR, DIST, CLEAR_RUN_CACHE: see ../measure_env.sh and +# ../split/run_split.sh. Give each side its own CORES and GPU. +set -u +S=${1:?usage: run_side.sh } +: "${ROOT:?set ROOT}" "${TL:?set TL}" "${CLIENT:?set CLIENT}" +SIDES=${SIDES:-before after} +case " $SIDES " in + *" $S "*) ;; + *) echo "side '$S' is not in SIDES='$SIDES'" >&2; exit 1 ;; +esac +SCRIPTS=$(dirname "$(dirname "$(realpath -- "${BASH_SOURCE[0]}")")") +export TL CLIENT +R=$ROOT/$S +mkdir -p "$R" +export TMPDIR=$R/tmp +mkdir -p "$TMPDIR" +log() { echo "$(date -u +%FT%TZ) $*" >> "$R/driver.log"; } +# Host conditions at each step, for judging how comparable the two sides are on a shared host. +snap() { + { + echo "== $1 $(date -u +%FT%TZ)" + uptime + rocm-smi -d "${GPU:-0}" --showuse --showmemuse --showpids 2>/dev/null | grep -Ev '^=+$|^$' + } >> "$R/conditions.log" +} + +# 1. CI-like split: wall time of each phase and the configs that pass +if ! grep -qs ALLDONE "$R/split/times.txt"; then + snap split_start; log split start + OUT=$R/split ART=$R/split_art TENSILE_CACHE=$R/cache_split/tensile \ + TENSILELITE_CACHE=$R/cache_split/tensilelite bash "$SCRIPTS/split/run_split.sh" + snap split_end; log split done +fi +python - "$R/split/run.xml" > "$R/split_run_passed.txt" <<'EOF' +import re +import sys +import xml.etree.ElementTree as ET + +for tc in ET.parse(sys.argv[1]).iter("testcase"): + m = re.search(r"test_config_run\[(.*)\]", tc.get("name", "")) + if m and not any(c.tag in ("failure", "error", "skipped") for c in tc): + print(m.group(1)) +EOF +touch "$R/split.done" + +# 2. shared config list: union of the configs whose run passed in any side's split run phase +for s in $SIDES; do + while [ ! -f "$ROOT/$s/split.done" ]; do sleep 30; done +done +for s in $SIDES; do cat "$ROOT/$s/split_run_passed.txt"; done | sort -u > "$R/configs.txt" +log "configs: $(wc -l < "$R/configs.txt")" + +# 3. per-config build, instrumented; caches cleared once at the start, artifacts kept +export ART=$R/art TENSILE_CACHE=$R/cache/tensile TENSILELITE_CACHE=$R/cache/tensilelite +snap build_start; log build start +OUT=$R/build SCRATCH=$R/scratch CLEAR_CACHE=once KEEP_ARTIFACTS=1 \ + bash "$SCRIPTS/per_config/run_per_config_build.sh" "$R/configs.txt" +snap build_end; log build done + +# 4. per-config run on those artifacts, caches not cleared +snap run_start; log run start +OUT=$R/run CLEAR_CACHE=never bash "$SCRIPTS/per_config/run_per_config.sh" "$R/configs.txt" +snap run_end; log run done diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/measure_env.sh b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/measure_env.sh new file mode 100644 index 000000000000..3d2134cf5933 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/measure_env.sh @@ -0,0 +1,93 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Sourced by split/run_split.sh, per_config/run_per_config.sh and per_config/run_per_config_build.sh. +# Every location the measurement scripts read, write or delete can be overridden from the +# environment; the defaults are the paths the committed baselines were measured with. +# TL TensileLite source tree pytest runs from (e.g. a git worktree of another commit) +# CLIENT prebuilt tensilelite-client +# ART build artifact dir (split build phase writes it, the run phases read it). The +# default is the measured baseline, which check_rm_target refuses to delete, so +# split/run_split.sh needs ART set to a new dir. +# TENSILE_CACHE Tensile's cache root, cleared by clear_caches. TENSILE_HELPER_CACHE_DIR, +# TENSILE_HELPER_PCH_DIR and TENSILE_HELPER_CHUNK_CACHE_DIR are always +# exported as $TENSILE_CACHE/helper_cache, $TENSILE_CACHE/helper_pch and +# $TENSILE_CACHE/helper_chunk_cache (inherited values are ignored), so clearing +# TENSILE_CACHE clears the helper-kernel cache, HIP-header PCH and helper chunk +# cache Tensile really uses. TENSILE_DISABLE_HELPER_PCH=1 (compiles without a +# PCH) and TENSILE_DISABLE_HELPER_CHUNK_CACHE=1 (compiles every helper chunk) +# are passed through. +# TENSILELITE_CACHE ~/.tensilelite, cleared by clear_caches as before (current TensileLite +# sources neither read nor write it). +# GPU_TARGET --gpu-targets of the build phases (default gfx950) +# CORES taskset core list (default 0-24; OMP_NUM_THREADS stays 25, so give 25 cores) +# GPU ROCR_VISIBLE_DEVICES of the runs (default 0) +# ANALYZE_DIR directory holding analyze_timing.py / aggregate_timing.py (default $TL/scripts), +# so logs of an older TL can be analysed with a newer analyzer +# CLEAR_CACHE per-config | once | never: when the per-config scripts clear the caches (default +# per-config; KEEP_CACHE=1 means once). once clears before the first config of the +# pass; never leaves the caches as the previous pass left them. +# OUT, LIST and SCRATCH are set by the scripts themselves (their defaults depend on the script); +# each script then calls check_rm_target on every directory it deletes. +TL=${TL:-/home/alvasile/rocm-libraries/projects/hipblaslt/tensilelite} +CLIENT=${CLIENT:-/tmp/tl_client_msgpack/tensilelite/client/tensilelite-client} +ART=${ART:-/tmp/tl_split_gfx950/artifacts} +TENSILE_CACHE=${TENSILE_CACHE:-/home/alvasile/.tensile} +TENSILELITE_CACHE=${TENSILELITE_CACHE:-/home/alvasile/.tensilelite} +GPU_TARGET=${GPU_TARGET:-gfx950} +CORES=${CORES:-0-24} +GPU=${GPU:-0} +ANALYZE_DIR=${ANALYZE_DIR:-$TL/scripts} +if [ -z "$CLEAR_CACHE" ]; then + if [ "${KEEP_CACHE:-0}" = "1" ]; then CLEAR_CACHE=once; else CLEAR_CACHE=per-config; fi +fi +case $CLEAR_CACHE in + per-config|once|never) ;; + *) echo "CLEAR_CACHE='$CLEAR_CACHE' must be per-config, once or never" >&2; exit 1 ;; +esac +export TENSILE_HELPER_CACHE_DIR=$TENSILE_CACHE/helper_cache +export TENSILE_HELPER_PCH_DIR=$TENSILE_CACHE/helper_pch +export TENSILE_HELPER_CHUNK_CACHE_DIR=$TENSILE_CACHE/helper_chunk_cache +MEASURE_DIR=$(dirname "$(realpath -- "${BASH_SOURCE[0]}")") +# Measured baseline client build and artifacts; never deleted. +BASELINE_DIRS=(/tmp/tl_client_msgpack /tmp/tl_split_gfx950/artifacts) + +# check_rm_target VAR...: exit unless every $VAR is safe to `rm -rf`: an absolute path at least two +# levels deep that is not $HOME or a directory containing it, and that neither is, contains, nor is +# inside $TL, $OUT, this script's directory, a baseline dir, $CLIENT's directory or $ART (unless +# VAR is ART). +check_rm_target() { + local var val real keep keeps + for var in "$@"; do + val=${!var} + case $val in + /*) ;; + *) echo "$var='$val' must be an absolute path" >&2; exit 1 ;; + esac + real=$(realpath -m -- "$val") + case $real in + /*/*) ;; + *) echo "$var='$val' is too broad to delete" >&2; exit 1 ;; + esac + if [ -n "$HOME" ]; then + keep=$(realpath -m -- "$HOME") + case $keep/ in + "$real"/*) echo "$var='$val' is or contains $keep; refusing to delete it" >&2; exit 1 ;; + esac + fi + keeps=("$TL" "$OUT" "$MEASURE_DIR" "${BASELINE_DIRS[@]}" "$(dirname -- "$CLIENT")") + [ "$var" = ART ] || keeps+=("$ART") + for keep in "${keeps[@]}"; do + [ -n "$keep" ] || continue + keep=$(realpath -m -- "$keep") + case $keep/ in + "$real"/*) echo "$var='$val' is or contains $keep; refusing to delete it" >&2; exit 1 ;; + esac + case $real/ in + "$keep"/*) echo "$var='$val' is inside $keep; refusing to delete it" >&2; exit 1 ;; + esac + done + done +} + +clear_caches() { rm -rf -- "$TENSILE_CACHE" "$TENSILELITE_CACHE"; } diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config.sh b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config.sh new file mode 100755 index 000000000000..b983a9e38e6a --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Run step of every gfx950 common config that passed in the split run phase, one at a time +# (serial, no xdist), with TimingInstrumentation on, and save per config: +# /.log.gz raw instrumented pytest/client log +# /_analysis.txt scripts/analyze_timing.py output +# /times.txt pytest wall seconds + return code +# Build artifacts come from the split build phase (../split) or from +# run_per_config_build.sh with KEEP_ARTIFACTS=1. Caches (~/.tensile, ~/.tensilelite) are cleared +# before each config, as in run_cold_warm.sh / the split phases, unless CLEAR_CACHE=once +# (or KEEP_CACHE=1): then they are cleared once before the first config and kept between configs +# (a normal run, where e.g. the helper-kernel cache is shared by all configs). CLEAR_CACHE=never +# keeps the cache a preceding build pass left (a CI-like build-then-run on one node). +# When every config is DONE, scripts/aggregate_timing.py sums the logs of ALL configs (failed ones +# included) into $OUT/AGGREGATE_analysis.txt. Re-running the script after a finished pass skips the +# runs and only regenerates the aggregate. +# Usage: bash run_per_config.sh [configs.txt] (one tensilelite/Tests/... yaml path per line) +# Env: TL, CLIENT, ART, TENSILE_CACHE, TENSILELITE_CACHE (see ../measure_env.sh), +# CORES, GPU, ANALYZE_DIR, CLEAR_CACHE (see ../measure_env.sh), +# OUT (default $TL/timing_runs/full_runs_gfx950/per_config), KEEP_CACHE. +source "$(dirname "${BASH_SOURCE[0]}")/../measure_env.sh" || exit 1 +OUT=${OUT:-$TL/timing_runs/full_runs_gfx950/per_config} +LIST=${1:-$OUT/configs.txt} +check_rm_target TENSILE_CACHE TENSILELITE_CACHE +cd "$TL" || exit 1 +mkdir -p "$OUT" +export OMP_NUM_THREADS=25 OPENBLAS_NUM_THREADS=25 ROCR_VISIBLE_DEVICES=$GPU PYTHONUNBUFFERED=1 +TO="--cxx-compiler,/usr/bin/amdclang++,--global-parameters,TimingInstrumentation=True" +COMMON=(-s -v -n 0 --timeout=5400 -m "common and not skip-gfx94X" + "--global-parameters=LibraryFormat='msgpack'" "--tensile-options=$TO" -p no:warnings -p no:cacheprovider) + +n=0; total=$(wc -l < "$LIST") +if [ "$CLEAR_CACHE" = once ]; then + clear_caches + echo "CLEAR_CACHE=once: caches cleared once at $(date -u +%FT%TZ)" >> "$OUT/progress.txt" +elif [ "$CLEAR_CACHE" = never ]; then + echo "CLEAR_CACHE=never: caches kept at $(date -u +%FT%TZ)" >> "$OUT/progress.txt" +fi +while read -r cfg; do + [ -z "$cfg" ] && continue + n=$((n+1)) + name=$(echo "${cfg#tensilelite/Tests/}" | sed -e 's/\.yaml$//' -e 's#/#__#g') + D=$OUT/$name + if [ -f "$D/times.txt" ] && grep -q DONE "$D/times.txt"; then continue; fi # resumable + mkdir -p "$D" + RID="tensilelite/Tests/common/test_config_run.py::test_config_run[$cfg]" + if [ "$CLEAR_CACHE" = per-config ]; then + clear_caches + fi + s=$(date +%s) + taskset -c "$CORES" pytest "$RID" --use-cache --artifact-dir "$ART" --prebuilt-client="$CLIENT" \ + "${COMMON[@]}" > "$D/$name.log" 2>&1 + rc=$? + echo "run rc=$rc seconds=$(( $(date +%s)-s )) ($n/$total)" > "$D/times.txt" + python "$ANALYZE_DIR/analyze_timing.py" "$D/$name.log" > "$D/${name}_analysis.txt" 2>&1 + gzip -f "$D/$name.log" + echo DONE >> "$D/times.txt" + echo "$(date -u +%T) $n/$total $cfg rc=$rc" >> "$OUT/progress.txt" +done < "$LIST" +python "$ANALYZE_DIR/aggregate_timing.py" "$OUT"/*/*.log.gz > "$OUT/AGGREGATE_analysis.txt" 2>&1 +echo "aggregate rc=$? logs=$(ls "$OUT"/*/*.log.gz | wc -l)" >> "$OUT/progress.txt" +echo ALLDONE > "$OUT/done.txt" diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config_build.sh b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config_build.sh new file mode 100644 index 000000000000..93f200f1ec96 --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/per_config/run_per_config_build.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# Build step of every common config, one at a time (serial, no xdist), with +# TimingInstrumentation on, COLD: the caches (~/.tensile, ~/.tensilelite) are cleared before every +# config, so each build pays the full helper-kernel / assembler / library cost. With +# CLEAR_CACHE=once they are cleared before the first config only (a CI-like build that starts +# with an empty cache). Counterpart of run_per_config.sh (run step). Saves per config: +# /_build.log.gz raw instrumented pytest log +# /_build_analysis.txt scripts/analyze_timing.py output +# /times.txt pytest wall seconds + return code +# Artifacts go to a per-config scratch dir that is deleted after the config's log is saved (they +# are not used by anything else; the run phase uses $ART). With KEEP_ARTIFACTS=1 they go to $ART +# instead (emptied once before the first config) and are kept, for run_per_config.sh. +# When every config is DONE, scripts/aggregate_timing.py sums the logs of ALL configs (failed ones +# included) into $OUT/AGGREGATE_analysis.txt. Re-running the script after a finished pass skips the +# builds and only regenerates the aggregate. +# Usage: bash run_per_config_build.sh [configs.txt] (one tensilelite/Tests/... yaml path per line) +# Env: TL, CLIENT, TENSILE_CACHE, TENSILELITE_CACHE (see ../measure_env.sh), +# GPU_TARGET, CORES, GPU, ANALYZE_DIR, CLEAR_CACHE (see ../measure_env.sh), KEEP_ARTIFACTS, ART, +# OUT (default .../per_config_build), SCRATCH (default /tmp/tl_build_scratch_gfx950) +source "$(dirname "${BASH_SOURCE[0]}")/../measure_env.sh" || exit 1 +OUT=${OUT:-$TL/timing_runs/full_runs_gfx950/per_config_build} +LIST=${1:-$TL/timing_runs/full_runs_gfx950/per_config/configs.txt} +SCRATCH=${SCRATCH:-/tmp/tl_build_scratch_gfx950} +check_rm_target TENSILE_CACHE TENSILELITE_CACHE SCRATCH +if [ "${KEEP_ARTIFACTS:-0}" = "1" ]; then check_rm_target ART; fi +if [ "$CLEAR_CACHE" = never ]; then + echo "CLEAR_CACHE=never is not supported by the build pass" >&2; exit 1 +fi +cd "$TL" || exit 1 +export OMP_NUM_THREADS=25 OPENBLAS_NUM_THREADS=25 ROCR_VISIBLE_DEVICES=$GPU PYTHONUNBUFFERED=1 +TO="--cxx-compiler,/usr/bin/amdclang++,--global-parameters,TimingInstrumentation=True" +COMMON=(-s -v -n 0 --timeout=5400 -m "common and not skip-gfx94X" + "--global-parameters=LibraryFormat='msgpack'" "--tensile-options=$TO" -p no:warnings -p no:cacheprovider) + +mkdir -p "$OUT" "$SCRATCH" +n=0; total=$(wc -l < "$LIST") +if [ ! -f "$OUT/started.txt" ]; then # first start of the pass, not a resume + if [ "$CLEAR_CACHE" = once ]; then + clear_caches + echo "CLEAR_CACHE=once: caches cleared once at $(date -u +%FT%TZ)" >> "$OUT/progress.txt" + fi + if [ "${KEEP_ARTIFACTS:-0}" = "1" ]; then rm -rf -- "$ART"; fi + date -u +%FT%TZ > "$OUT/started.txt" +fi +if [ "${KEEP_ARTIFACTS:-0}" = "1" ]; then mkdir -p "$ART"; fi +while read -r cfg; do + [ -z "$cfg" ] && continue + n=$((n+1)) + name=$(echo "${cfg#tensilelite/Tests/}" | sed -e 's/\.yaml$//' -e 's#/#__#g') + D=$OUT/$name + if [ -f "$D/times.txt" ] && grep -q DONE "$D/times.txt"; then continue; fi # resumable + mkdir -p "$D" + A=$SCRATCH/$name + if [ "${KEEP_ARTIFACTS:-0}" = "1" ]; then A=$ART; else rm -rf -- "$A"; fi + BID="tensilelite/Tests/common/test_config_build.py::test_config_build[$cfg]" + if [ "$CLEAR_CACHE" = per-config ]; then clear_caches; fi # cold cache for every build + s=$(date +%s) + taskset -c "$CORES" pytest "$BID" --build-only --gpu-targets "$GPU_TARGET" --artifact-dir "$A" \ + --prebuilt-client="$CLIENT" "${COMMON[@]}" > "$D/${name}_build.log" 2>&1 # ductile builds run the client + rc=$? + echo "build_cold rc=$rc seconds=$(( $(date +%s)-s )) ($n/$total)" > "$D/times.txt" + python "$ANALYZE_DIR/analyze_timing.py" "$D/${name}_build.log" > "$D/${name}_build_analysis.txt" 2>&1 + gzip -f "$D/${name}_build.log" + if [ "${KEEP_ARTIFACTS:-0}" != "1" ]; then rm -rf -- "$A"; fi + echo DONE >> "$D/times.txt" + echo "$(date -u +%T) $n/$total $cfg rc=$rc" >> "$OUT/progress.txt" +done < "$LIST" +python "$ANALYZE_DIR/aggregate_timing.py" "$OUT"/*/*_build.log.gz > "$OUT/AGGREGATE_analysis.txt" 2>&1 +echo "aggregate rc=$? logs=$(ls "$OUT"/*/*_build.log.gz | wc -l)" >> "$OUT/progress.txt" +echo ALLDONE > "$OUT/done.txt" diff --git a/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/split/run_split.sh b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/split/run_split.sh new file mode 100755 index 000000000000..dda6095f0a5b --- /dev/null +++ b/projects/hipblaslt/tensilelite/timing_runs/full_runs_gfx950/split/run_split.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +# +# gfx950 build / run split of the full common set (build on --build-only, then --use-cache run), +# same conditions as ../../full_runs/run.sh: 25 cores, 4 xdist workers, 5400s per-test timeout, +# msgpack client. Gives per-test RUN-phase durations, which is what a client-launch-batching change +# can affect (the combined pass is dominated by kernel builds). +# Env: TL, CLIENT, ART, TENSILE_CACHE, TENSILELITE_CACHE (see ../measure_env.sh), +# GPU_TARGET, CORES, GPU (see ../measure_env.sh), +# OUT (default $TL/timing_runs/full_runs_gfx950/split). ART is deleted and rebuilt, so it must +# be set: the default ART is the measured baseline, which the script refuses to delete. +# DIST pytest-xdist scheduler (default worksteal, as the hw-common CI category; the +# committed split numbers were measured with load, xdist's default) +# CLEAR_RUN_CACHE 1 also clears the caches before the run phase (default 0: the run phase +# starts from the cache the build phase left; the committed split numbers were +# measured with 1). The build phase always starts from empty caches. +source "$(dirname "${BASH_SOURCE[0]}")/../measure_env.sh" || exit 1 +OUT=${OUT:-$TL/timing_runs/full_runs_gfx950/split} +DIST=${DIST:-worksteal} +CLEAR_RUN_CACHE=${CLEAR_RUN_CACHE:-0} +case $CLEAR_RUN_CACHE in + 0|1) ;; + *) echo "CLEAR_RUN_CACHE='$CLEAR_RUN_CACHE' must be 0 or 1" >&2; exit 1 ;; +esac +check_rm_target TENSILE_CACHE TENSILELITE_CACHE ART +cd "$TL" || exit 1 +mkdir -p "$OUT" +export OMP_NUM_THREADS=25 OPENBLAS_NUM_THREADS=25 ROCR_VISIBLE_DEVICES=$GPU PYTHONUNBUFFERED=1 +COMMON=(tensilelite/Tests/common -v --durations=0 --timeout=5400 -n 4 --dist="$DIST" -m "common and not skip-gfx94X" + "--global-parameters=LibraryFormat='msgpack'" "--tensile-options=--cxx-compiler,/usr/bin/amdclang++") +phase() { # name, clear caches first (0|1), extra args... + local name=$1 clear=$2; shift 2 + if [ "$clear" = 1 ]; then clear_caches; fi + echo "START $name dist=$DIST caches_cleared=$clear $(date -u +%FT%TZ)" >> "$OUT/times.txt" + local s=$(date +%s) + taskset -c "$CORES" pytest "${COMMON[@]}" "$@" --junit-xml="$OUT/$name.xml" > "$OUT/$name.log" 2>&1 + local rc=$? + echo "END $name rc=$rc seconds=$(( $(date +%s)-s )) $(date -u +%FT%TZ)" >> "$OUT/times.txt" +} +rm -rf -- "$ART"; mkdir -p "$ART" +phase build 1 --build-only --gpu-targets "$GPU_TARGET" --artifact-dir "$ART" --prebuilt-client="$CLIENT" +du -sh "$ART" >> "$OUT/times.txt" +phase run "$CLEAR_RUN_CACHE" --use-cache --artifact-dir "$ART" --prebuilt-client="$CLIENT" +echo ALLDONE >> "$OUT/times.txt"