[hipblaslt] perf-script support running a sh file commands

projects/hipblaslt/clients/scripts/performance/bench.py

@@ -119,3 +119,99 @@ async def run_command(*args, benchType=benchType, timeout=None):
     loop.close()
 
     return csvKeys, benchResultsList, success
+
+
+#####################################
+# For ./hipblaslt-perf --run_sh
+#####################################
+def run_sh_cmd(cmdLine,
+               verbose=False,
+               timeout=300):
+    """Run single bench from sh"""
+
+    cmd = cmdLine.split(' ')
+    cmd = [str(x) for x in cmd]
+    logging.info('running: ' + ' '.join(cmd))
+    if verbose:
+        print('running: ' + ' '.join(cmd))
+
+    startingToken = "["
+    solNameToken = "--Solution name:"
+    csvKeys = []
+    benchResultsList = {}
+    capturingValues = False
+
+    async def run_command(*args, timeout=None):
+
+        process = await asyncio.create_subprocess_exec(
+            *args, stdout=asyncio.subprocess.PIPE)
+
+        nonlocal startingToken
+        nonlocal csvKeys
+        nonlocal benchResultsList
+        nonlocal capturingValues
+
+        singleValuesList = []
+        solutionName = "N/A"
+
+        while True:
+            try:
+                line = await asyncio.wait_for(process.stdout.readline(),
+                                              timeout)
+            except asyncio.TimeoutError:
+                logging.info(
+                    "timeout expired. killed. Please check the process.")
+                print("timeout expired. killed. Please check the process.")
+                process.kill()  # Timeout or some criterion is not satisfied
+                break
+
+            if not line:
+                break
+            else:
+                line = line.decode('utf-8').rstrip('\n')
+                line = line.strip()
+
+                # capturing values right after capturing keys
+                if capturingValues:
+                    singleValuesList = line.split(',')
+                    # default is empty if --print_kernel_info is not in the bench cmd
+                    solutionName = "N/A"
+                    capturingValues = False
+                    continue
+
+                if line.startswith(startingToken):
+                    line = line.replace('hipblaslt-Gflops', 'gflops')
+                    line = line.replace('hipblaslt-GB/s', 'GB/s')
+                    splitLine = line.split(':')
+                    # SSN = splitLine[0] # should be [0]
+                    keys = splitLine[1] + str(",solution-name")
+                    csvKeys = keys.split(',')
+                    # print(f'\n{keys}')
+                    # print(f'\n{csvKeys}')
+                    capturingValues = True # Next line must be values
+                else:
+                    # if is "--Solution name:" (--print_kernel_info), then we capture this
+                    if line.startswith(solNameToken):
+                        splitLine = line.split(':')
+                        solutionName = splitLine[1].strip()
+                    # simply ignore irrelative msg
+                    else:
+                        continue
+
+        singleValuesList.append(solutionName)
+        benchResultsList = defaultdict(str, zip(csvKeys, singleValuesList))
+
+        return await process.wait()  # Wait for the child process to exit
+
+    if sys.platform == "win32":
+        loop = asyncio.ProactorEventLoop()  # For subprocess' pipes on Windows
+        asyncio.set_event_loop(loop)
+    else:
+        loop = asyncio.new_event_loop()
+
+    returncode = loop.run_until_complete(run_command(*cmd, timeout=timeout))
+    success = returncode == 0
+
+    loop.close()
+
+    return csvKeys, benchResultsList, success

projects/hipblaslt/clients/scripts/performance/generator.py

@@ -73,4 +73,45 @@ def __post_init__(self):
 
     def generate_problemSet(self):
         for g in self.suites.values():
-            yield from g()
+            yield from g()
+
+@dataclass
+class ShellScriptProblemGenerator:
+    sh_filename: str = None
+    bench_exec: str = None
+    benchCMDs: List[str] = field(default_factory=list)
+
+    def __post_init__(self):
+        # print("sh file: " + str(self.sh_filename))
+        # print("bench_exec: " + str(self.bench_exec))
+        # load file and readline
+        try:
+            with open(self.sh_filename, 'r') as sh_file:
+                # Iterate over each line in the file
+                for cmd in sh_file:
+                    cmd = cmd.strip()
+                    # skip invalid line
+                    if cmd.startswith("#") or cmd.count("hipblaslt-bench") == 0:
+                        continue
+                    # --verfiy will output other values, not supported yet
+                    if cmd.count("verfiy") > 0 or cmd.count("-v") > 0:
+                        print(f'--verify or -v is not supported, skip bench.')
+                        continue
+                    # TODO- not supported for offline tuning
+                    if cmd.count("requested_solution") > 0:
+                        print(f'--requested_solution is not supported, skip bench.')
+                        continue
+
+                    cmd = cmd.replace("./hipblaslt-bench", str(self.bench_exec))
+                    # print("parsed cmd: " + cmd)
+                    self.benchCMDs.append(cmd)
+
+        except FileNotFoundError:
+            print(f'Error: The file {self.sh_filename} was not found.')
+
+        except Exception as e:
+            print(f'An error occurred: {e}')
+
+    def iterate_cmd(self):
+        for cmd in self.benchCMDs:
+            yield cmd

projects/hipblaslt/clients/scripts/performance/hipblaslt-perf

@@ -28,7 +28,7 @@ import logging
 import bench
 from dataclasses import dataclass, field
 from bench_sample import BenchSample, MeasurementKey
-from generator import SuiteProblemGenerator
+from generator import SuiteProblemGenerator, ShellScriptProblemGenerator
 from pathlib import Path
 from git_info import create_github_file
 from specs import get_machine_specs
@@ -139,8 +139,74 @@ def runBenchmark(benchType, probBenchResults:Dict[str, BenchSample], prob_args,
         if finalize:
             print(content)
 
-def command_perf(arguments, probYaml_foler):
-    """Run bench"""
+def run_sh_bench(arguments):
+    """Run bench cmds from a .sh file"""
+
+    if arguments.workspace:
+        print(f'Output data to {arguments.workspace}')
+    else:
+        print("Workspace not set. use -w /path/of/workspace")
+        return
+
+    if arguments.execFolder:
+        print(f'Will call the hipblaslt-bench executable from folder {arguments.execFolder}')
+    else:
+        print("execFolder not set. use -e /path/of/execFolder")
+        return
+
+    sh_filepath = pathlib.Path(os.path.join(os.path.curdir, arguments.run_sh)).resolve()
+    if os.path.exists(sh_filepath):
+        print(f'Will run the bench commands from file: {sh_filepath}')
+    else:
+        print(f'Input sh filepath {sh_filepath} is not existing, abort. Please check')
+        return
+
+    out_folder = arguments.workspace
+    exec_folder = arguments.execFolder
+    bench_exec_path = os.path.join(exec_folder, "hipblaslt-bench")
+    generator = ShellScriptProblemGenerator(sh_filepath, bench_exec_path)
+
+    print(f'Note: When running with --run_sh, argument --suite, --tag, --samples and --pts will be ignored')
+
+    subDirectory = os.path.join(out_folder, "bench_output_csv")
+    Path(subDirectory).mkdir(parents=True, exist_ok=True)
+    filename, _ = os.path.splitext(os.path.basename(arguments.run_sh))
+
+    if arguments.csvfile is not None:
+        out_csv_filename = os.path.join(subDirectory, arguments.csvfile + '.csv')
+    else:
+        out_csv_filename = os.path.join(subDirectory, filename + '.csv')
+        print(f'Info: --csvfile not set, use sh-filename as csv-filename')
+    # print(f'Info: run_sh = {arguments.run_sh}')
+    # print(f'Info: sh filename = {filename}')
+    print(f'Info: Output csv file = {out_csv_filename}')
+
+    logging.info("Start bench.")
+
+    print("\n==================================\n|| Running bench sh file {}\n==================================".format(arguments.run_sh))
+
+    content = ''
+    csvKeys = ''
+    writeHeader = True
+    csv_file = open(out_csv_filename, 'w')
+
+    for singleCMD in generator.iterate_cmd():
+        # print("singleCMD: " + singleCMD)
+        csvKeys, benchResults, success = bench.run_sh_cmd(singleCMD, True)
+        content = extractProblemDescStr(benchResults, csvKeys) + '\n'
+        if writeHeader:
+            header = ','.join([str(key) for key in csvKeys]) + '\n'
+            csv_file.write(header)
+            writeHeader = False
+        csv_file.write(content)
+
+    print(f'\nResults written to {csv_file.name}')
+    csv_file.close()
+
+    logging.info("Finish bench.")
+
+def run_suite_bench(arguments, probYaml_foler):
+    """Run suite bench"""
 
     if arguments.workspace:
         print(f'Output data to {arguments.workspace}')
@@ -191,7 +257,7 @@ def command_perf(arguments, probYaml_foler):
                 writeCSVHeader = False
 
         if csv_file is not None:
-            print("\nResults written to {}".format(csv_file.name))
+            print(f'\nResults written to {csv_file.name}')
             csv_file.close()
 
         # check if we need to output other files for pts
@@ -212,9 +278,9 @@ def main():
     dir_of_this_repo = pathlib.Path(os.path.join(dir_of_this_script, '../../../')).resolve()
     probYaml_folder = pathlib.Path(os.path.join(dir_of_this_script, 'problems/')).resolve()
     executable_folder = pathlib.Path(os.path.join(dir_of_this_repo, "build/release/clients/staging/")).resolve()
-    print(f'info: path of this script = {dir_of_this_script}')
-    print(f'info: path of this repos = {dir_of_this_repo}')
-    print(f'info: path of bench yaml problems = {probYaml_folder}')
+    print(f'Info: path of this script = {dir_of_this_script}')
+    print(f'Info: path of this repos = {dir_of_this_repo}')
+    print(f'Info: path of bench yaml problems = {probYaml_folder}')
 
     parser.add_argument('-w',
                         '--workspace',
@@ -229,6 +295,16 @@ def main():
                         help='folder where the cpp bench executables are located',
                         default=executable_folder)
 
+    parser.add_argument('--run_sh',
+                        type=str,
+                        help='run a sh file calling a set of ./hipblaslt-bench commands',
+                        default=None)
+
+    parser.add_argument('--csvfile',
+                        type=str,
+                        help='specify the output csv filename (without ext) when --run_sh, if not set, use [sh-filename].csv',
+                        default=None)
+
     parser.add_argument('--suite',
                         type=str,
                         action='append')
@@ -255,7 +331,10 @@ def main():
 
     arguments = parser.parse_args()
 
-    command_perf(arguments, probYaml_folder)
+    if arguments.run_sh is not None:
+        run_sh_bench(arguments)
+    else:
+        run_suite_bench(arguments, probYaml_folder)
 
     sys.exit(0)