diff --git a/projects/hipblaslt/clients/scripts/performance/bench.py b/projects/hipblaslt/clients/scripts/performance/bench.py index f1e2e6d49e4..279e6e64ffb 100644 --- a/projects/hipblaslt/clients/scripts/performance/bench.py +++ b/projects/hipblaslt/clients/scripts/performance/bench.py @@ -119,3 +119,99 @@ async def run_command(*args, benchType=benchType, timeout=None): loop.close() return csvKeys, benchResultsList, success + + +##################################### +# For ./hipblaslt-perf --run_sh +##################################### +def run_sh_cmd(cmdLine, + verbose=False, + timeout=300): + """Run single bench from sh""" + + cmd = cmdLine.split(' ') + cmd = [str(x) for x in cmd] + logging.info('running: ' + ' '.join(cmd)) + if verbose: + print('running: ' + ' '.join(cmd)) + + startingToken = "[" + solNameToken = "--Solution name:" + csvKeys = [] + benchResultsList = {} + capturingValues = False + + async def run_command(*args, timeout=None): + + process = await asyncio.create_subprocess_exec( + *args, stdout=asyncio.subprocess.PIPE) + + nonlocal startingToken + nonlocal csvKeys + nonlocal benchResultsList + nonlocal capturingValues + + singleValuesList = [] + solutionName = "N/A" + + while True: + try: + line = await asyncio.wait_for(process.stdout.readline(), + timeout) + except asyncio.TimeoutError: + logging.info( + "timeout expired. killed. Please check the process.") + print("timeout expired. killed. Please check the process.") + process.kill() # Timeout or some criterion is not satisfied + break + + if not line: + break + else: + line = line.decode('utf-8').rstrip('\n') + line = line.strip() + + # capturing values right after capturing keys + if capturingValues: + singleValuesList = line.split(',') + # default is empty if --print_kernel_info is not in the bench cmd + solutionName = "N/A" + capturingValues = False + continue + + if line.startswith(startingToken): + line = line.replace('hipblaslt-Gflops', 'gflops') + line = line.replace('hipblaslt-GB/s', 'GB/s') + splitLine = line.split(':') + # SSN = splitLine[0] # should be [0] + keys = splitLine[1] + str(",solution-name") + csvKeys = keys.split(',') + # print(f'\n{keys}') + # print(f'\n{csvKeys}') + capturingValues = True # Next line must be values + else: + # if is "--Solution name:" (--print_kernel_info), then we capture this + if line.startswith(solNameToken): + splitLine = line.split(':') + solutionName = splitLine[1].strip() + # simply ignore irrelative msg + else: + continue + + singleValuesList.append(solutionName) + benchResultsList = defaultdict(str, zip(csvKeys, singleValuesList)) + + return await process.wait() # Wait for the child process to exit + + if sys.platform == "win32": + loop = asyncio.ProactorEventLoop() # For subprocess' pipes on Windows + asyncio.set_event_loop(loop) + else: + loop = asyncio.new_event_loop() + + returncode = loop.run_until_complete(run_command(*cmd, timeout=timeout)) + success = returncode == 0 + + loop.close() + + return csvKeys, benchResultsList, success \ No newline at end of file diff --git a/projects/hipblaslt/clients/scripts/performance/generator.py b/projects/hipblaslt/clients/scripts/performance/generator.py index d8de7f21d61..bd0823dd900 100644 --- a/projects/hipblaslt/clients/scripts/performance/generator.py +++ b/projects/hipblaslt/clients/scripts/performance/generator.py @@ -73,4 +73,45 @@ def __post_init__(self): def generate_problemSet(self): for g in self.suites.values(): - yield from g() \ No newline at end of file + yield from g() + +@dataclass +class ShellScriptProblemGenerator: + sh_filename: str = None + bench_exec: str = None + benchCMDs: List[str] = field(default_factory=list) + + def __post_init__(self): + # print("sh file: " + str(self.sh_filename)) + # print("bench_exec: " + str(self.bench_exec)) + # load file and readline + try: + with open(self.sh_filename, 'r') as sh_file: + # Iterate over each line in the file + for cmd in sh_file: + cmd = cmd.strip() + # skip invalid line + if cmd.startswith("#") or cmd.count("hipblaslt-bench") == 0: + continue + # --verfiy will output other values, not supported yet + if cmd.count("verfiy") > 0 or cmd.count("-v") > 0: + print(f'--verify or -v is not supported, skip bench.') + continue + # TODO- not supported for offline tuning + if cmd.count("requested_solution") > 0: + print(f'--requested_solution is not supported, skip bench.') + continue + + cmd = cmd.replace("./hipblaslt-bench", str(self.bench_exec)) + # print("parsed cmd: " + cmd) + self.benchCMDs.append(cmd) + + except FileNotFoundError: + print(f'Error: The file {self.sh_filename} was not found.') + + except Exception as e: + print(f'An error occurred: {e}') + + def iterate_cmd(self): + for cmd in self.benchCMDs: + yield cmd \ No newline at end of file diff --git a/projects/hipblaslt/clients/scripts/performance/hipblaslt-perf b/projects/hipblaslt/clients/scripts/performance/hipblaslt-perf index 18b9e4db214..879a9e824c7 100755 --- a/projects/hipblaslt/clients/scripts/performance/hipblaslt-perf +++ b/projects/hipblaslt/clients/scripts/performance/hipblaslt-perf @@ -28,7 +28,7 @@ import logging import bench from dataclasses import dataclass, field from bench_sample import BenchSample, MeasurementKey -from generator import SuiteProblemGenerator +from generator import SuiteProblemGenerator, ShellScriptProblemGenerator from pathlib import Path from git_info import create_github_file from specs import get_machine_specs @@ -139,8 +139,74 @@ def runBenchmark(benchType, probBenchResults:Dict[str, BenchSample], prob_args, if finalize: print(content) -def command_perf(arguments, probYaml_foler): - """Run bench""" +def run_sh_bench(arguments): + """Run bench cmds from a .sh file""" + + if arguments.workspace: + print(f'Output data to {arguments.workspace}') + else: + print("Workspace not set. use -w /path/of/workspace") + return + + if arguments.execFolder: + print(f'Will call the hipblaslt-bench executable from folder {arguments.execFolder}') + else: + print("execFolder not set. use -e /path/of/execFolder") + return + + sh_filepath = pathlib.Path(os.path.join(os.path.curdir, arguments.run_sh)).resolve() + if os.path.exists(sh_filepath): + print(f'Will run the bench commands from file: {sh_filepath}') + else: + print(f'Input sh filepath {sh_filepath} is not existing, abort. Please check') + return + + out_folder = arguments.workspace + exec_folder = arguments.execFolder + bench_exec_path = os.path.join(exec_folder, "hipblaslt-bench") + generator = ShellScriptProblemGenerator(sh_filepath, bench_exec_path) + + print(f'Note: When running with --run_sh, argument --suite, --tag, --samples and --pts will be ignored') + + subDirectory = os.path.join(out_folder, "bench_output_csv") + Path(subDirectory).mkdir(parents=True, exist_ok=True) + filename, _ = os.path.splitext(os.path.basename(arguments.run_sh)) + + if arguments.csvfile is not None: + out_csv_filename = os.path.join(subDirectory, arguments.csvfile + '.csv') + else: + out_csv_filename = os.path.join(subDirectory, filename + '.csv') + print(f'Info: --csvfile not set, use sh-filename as csv-filename') + # print(f'Info: run_sh = {arguments.run_sh}') + # print(f'Info: sh filename = {filename}') + print(f'Info: Output csv file = {out_csv_filename}') + + logging.info("Start bench.") + + print("\n==================================\n|| Running bench sh file {}\n==================================".format(arguments.run_sh)) + + content = '' + csvKeys = '' + writeHeader = True + csv_file = open(out_csv_filename, 'w') + + for singleCMD in generator.iterate_cmd(): + # print("singleCMD: " + singleCMD) + csvKeys, benchResults, success = bench.run_sh_cmd(singleCMD, True) + content = extractProblemDescStr(benchResults, csvKeys) + '\n' + if writeHeader: + header = ','.join([str(key) for key in csvKeys]) + '\n' + csv_file.write(header) + writeHeader = False + csv_file.write(content) + + print(f'\nResults written to {csv_file.name}') + csv_file.close() + + logging.info("Finish bench.") + +def run_suite_bench(arguments, probYaml_foler): + """Run suite bench""" if arguments.workspace: print(f'Output data to {arguments.workspace}') @@ -191,7 +257,7 @@ def command_perf(arguments, probYaml_foler): writeCSVHeader = False if csv_file is not None: - print("\nResults written to {}".format(csv_file.name)) + print(f'\nResults written to {csv_file.name}') csv_file.close() # check if we need to output other files for pts @@ -212,9 +278,9 @@ def main(): dir_of_this_repo = pathlib.Path(os.path.join(dir_of_this_script, '../../../')).resolve() probYaml_folder = pathlib.Path(os.path.join(dir_of_this_script, 'problems/')).resolve() executable_folder = pathlib.Path(os.path.join(dir_of_this_repo, "build/release/clients/staging/")).resolve() - print(f'info: path of this script = {dir_of_this_script}') - print(f'info: path of this repos = {dir_of_this_repo}') - print(f'info: path of bench yaml problems = {probYaml_folder}') + print(f'Info: path of this script = {dir_of_this_script}') + print(f'Info: path of this repos = {dir_of_this_repo}') + print(f'Info: path of bench yaml problems = {probYaml_folder}') parser.add_argument('-w', '--workspace', @@ -229,6 +295,16 @@ def main(): help='folder where the cpp bench executables are located', default=executable_folder) + parser.add_argument('--run_sh', + type=str, + help='run a sh file calling a set of ./hipblaslt-bench commands', + default=None) + + parser.add_argument('--csvfile', + type=str, + help='specify the output csv filename (without ext) when --run_sh, if not set, use [sh-filename].csv', + default=None) + parser.add_argument('--suite', type=str, action='append') @@ -255,7 +331,10 @@ def main(): arguments = parser.parse_args() - command_perf(arguments, probYaml_folder) + if arguments.run_sh is not None: + run_sh_bench(arguments) + else: + run_suite_bench(arguments, probYaml_folder) sys.exit(0)