From 99689c3f2a531219972028e2f00d2f1bdfeae031 Mon Sep 17 00:00:00 2001 From: James Newling Date: Thu, 27 Aug 2026 11:48:22 -0700 Subject: [PATCH] chore(tensilelite): remove obsolete benchmark client module --- .../Tensile/TensileBenchmarkLibraryClient.py | 183 ------------- .../test_benchclient_char.py | 247 ------------------ .../test_stats_char.py | 30 --- .../characterization/coverage-baseline.json | 1 - 4 files changed, 461 deletions(-) delete mode 100644 projects/hipblaslt/tensilelite/Tensile/TensileBenchmarkLibraryClient.py delete mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_benchclient_char.py delete mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_stats_char.py diff --git a/projects/hipblaslt/tensilelite/Tensile/TensileBenchmarkLibraryClient.py b/projects/hipblaslt/tensilelite/Tensile/TensileBenchmarkLibraryClient.py deleted file mode 100644 index eafef8d2222d..000000000000 --- a/projects/hipblaslt/tensilelite/Tensile/TensileBenchmarkLibraryClient.py +++ /dev/null @@ -1,183 +0,0 @@ -################################################################################ -# -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -import sys -import csv -import os -from subprocess import Popen, PIPE - -################################################################################ -# Benchmark Problem Size -################################################################################ -def BenchmarkProblemSize(cmdPrefix, row): - cmd = cmdPrefix - cmd += " --sizes" - for size in row: - cmd += " " - cmd += size.lstrip() - sys.stderr.write(cmd) - sys.stderr.write("\n") - process = Popen(cmd, stdout=PIPE, shell=True) - stdout = process.communicate()[0] - sys.stderr.write(stdout) - - # find beginning of data - initializingIdx = stdout.find("Initializing") - newLineIdx = stdout.find("\n", initializingIdx) - stdout = stdout[newLineIdx+1:] - - gflopList = [] - msList = [] - - # skip first b/c warmup/lookup - newLineIdx = stdout.find("\n") - stdout = stdout[newLineIdx+1:] - - # parse every line of data - while "\n" in stdout: - newLineIdx = stdout.find("\n") - lineString = stdout[:newLineIdx] - splits = lineString.split(",") - if len(splits) < 3: - break - gflopsString = splits[1].lstrip() - msString = splits[3].lstrip() - gflops = float(gflopsString) - ms = float(msString) - gflopList.append(gflops) - msList.append(ms) - - # next line - stdout = stdout[newLineIdx+1:] - return (gflopList, msList) - - -################################################################################ -# Print Stats -################################################################################ -def PrintStats(header, row, gflopList, msList): - meanGFlops = mean(gflopList) - meanMs = mean(msList) - medianGFlops = median(gflopList) - medianMs = median(msList) - stddevGFlops = stddev(gflopList) / meanGFlops - stddevMs = stddev(msList) / meanMs - - # format output - line = "" - for size in row: - line += "%6u, " % int(size) - line += "%9.2f, %9.4f, " % (medianGFlops, medianMs) - line += "%9.2f, %9.4f, " % (meanGFlops, meanMs) - line += "%9.5f, %9.5f" % (stddevGFlops, stddevMs) - sys.stdout.write(line) - sys.stdout.write("\n") - sys.stdout.flush() - sys.stderr.write("[STDOUT] %s\n" % header) - sys.stderr.write("[STDOUT] %s\n" % line) - sys.stderr.write("[END]\n\n\n") - sys.stderr.flush() - - -################################################################################ -# Benchmark Problem Sizes -################################################################################ -def TensileBenchmarkLibraryClient(userArgs): - if len(userArgs) < 2: - line = "USAGE: python TensileBenchmarkLibraryClient.py sizes.csv library_client_command > name.txt 2> name.raw.txt \n" - sys.stderr.write(line) - line = "Example: python TensileBenchmarkLibraryClient.py sizes.csv ./4_LibraryClient/build/client --function-idx 1 --num-benchmarks 100 --use-gpu-timer 0 > nn.txt 2> nn.raw.txt\n" - sys.stderr.write(line) - line = "[BE SURE TO PIN CLOCKS]\n" - sys.stderr.write(line) - exit(-1) - - # parse problem sizes path - problemSizesPath = os.path.realpath( userArgs[0] ) - line = "ProblemSizesPath: %s\n" % problemSizesPath - sys.stdout.write(line) - sys.stderr.write(line) - - # parse library client command - libraryClientCommandList = userArgs[1:] - libraryClientCommand = " ".join(libraryClientCommandList) - line = "LibraryClientCommand: %s\n" % libraryClientCommand - sys.stdout.write(line) - sys.stdout.write("\n") - sys.stderr.write(line) - sys.stderr.write("\n\n") - - # read problem sizes file - csvFileRaw = open(problemSizesPath, "r") - csvFile = csv.reader(csvFileRaw) - - # column headers - numIndices = -1 - for row in csvFile: - numIndices = len(row) - firstRow = row - break - header = "" - for i in range(0, numIndices): - sizeStr = "size%u" % i - header += "%6s, " % sizeStr - header += "%9s, %9s, " % ( "medianGF", "medianMs" ) - header += "%9s, %9s, " % ( "meanGF", "meanMs" ) - header += "%9s, %9s" % ( "rstdGF", "rstdMs" ) - sys.stdout.write(header) - sys.stdout.write("\n") - - # benchmark each problem size - for row in [firstRow]: - (gflopList, msList) = BenchmarkProblemSize(libraryClientCommand, row) - PrintStats(header, row, gflopList, msList) - - # benchmark each problem size - for row in csvFile: - (gflopList, msList) = BenchmarkProblemSize(libraryClientCommand, row) - PrintStats(header, row, gflopList, msList) - -def median(lst): - sortedList = sorted(lst) - return sortedList[len(sortedList)/2] - -def mean(lst): - total = 0 - for i in range(len(lst)): - total += lst[i] - return (total / len(lst)) - -def stddev(lst): - total = 0 - mn = mean(lst) - for i in range(len(lst)): - total += pow((lst[i]-mn),2) - return (total/(len(lst)-1))**0.5 - -# installed "tensileBenchmarkLibraryClient" command -def main(): - TensileBenchmarkLibraryClient(sys.argv[1:]) - -if __name__ == "__main__": - main() diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_benchclient_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_benchclient_char.py deleted file mode 100644 index 7d8ae90f90c2..000000000000 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_benchclient_char.py +++ /dev/null @@ -1,247 +0,0 @@ -################################################################################ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -################################################################################ - -"""Characterization tests for Tensile.TensileBenchmarkLibraryClient. - -This module pins the client benchmark driver entry points and the working stats -helpers (mean/stddev). The median and PrintStats functions are pinned in existing -test_stats_char.py and cannot be exercised further without fixing the Python 2/3 -division bug in source (outside ADD-ONLY scope). - -The main TensileBenchmarkLibraryClient function is tested via entry-point paths -that exercise CSV reading, argument validation, and path assembly. The path through -BenchmarkProblemSize (subprocess launch / stdout parsing) is realistically tested -via the end-to-end main() test with a CPU-only mock script. -""" - -import importlib -import io -import sys -import tempfile -from pathlib import Path - -import pytest - -pytestmark = pytest.mark.unit - -M = importlib.import_module("Tensile.TensileBenchmarkLibraryClient") - - -# ============================================================================ -# CPU-Only Test Fixtures -# ============================================================================ - - -@pytest.fixture -def mock_csv_file(tmp_path): - """Create a temporary CSV file with benchmark problem sizes. - - CSV format: each row is a problem size (comma-separated integers). - """ - csv_path = tmp_path / "sizes.csv" - csv_content = """128,128,1,512 -256,256,1,1024 -""" - csv_path.write_text(csv_content) - return str(csv_path) - - -@pytest.fixture -def mock_client_script(tmp_path): - """Create a CPU-only fake client script that outputs data in the expected format. - - The script mimics the real client output: - - A preamble with "Initializing" - - A warmup line (skipped) - - Data lines: "idx,GFlops,X,Time(ms),Y,Z" - """ - script_path = tmp_path / "fake_client.sh" - script_content = """#!/bin/bash -echo "Initializing device..." -echo "Warmup run" -echo "0,500.00,x,1.0000,y,z" -echo "1,550.00,x,0.9500,y,z" -echo "2,480.00,x,1.0500,y,z" -""" - script_path.write_text(script_content) - script_path.chmod(0o755) - return str(script_path) - - -# ============================================================================ -# Unit Tests: Working Stats Helpers (coverage lines 165-169, 171-176) -# ============================================================================ - - -def test_mean_basic(): - """Pin mean([2, 4, 6]) == 4. Covers lines 166-169.""" - assert M.mean([2, 4, 6]) == 4 - - -def test_mean_single(): - """Pin mean([5]) == 5. Covers lines 166-169.""" - assert M.mean([5]) == 5 - - -def test_stddev_basic(): - """Pin sample stddev([2, 4, 6]) == 2.0. Covers lines 171-176.""" - assert M.stddev([2, 4, 6]) == 2.0 - - -def test_median_py3_bug(): - """PIN LATENT BUG: median([3,1,2]) raises TypeError in Python 3 - because len(sortedList)/2 yields a float, not a valid list index. - - This bug is already pinned in test_stats_char.py; we re-pin it here - to document that median() cannot be called in end-to-end tests. - Covered by existing test_stats_char.py::test_median_is_broken_in_py3. - """ - with pytest.raises(TypeError): - M.median([3, 1, 2]) - - -# ============================================================================ -# Integration Test: Error Paths (coverage lines 107-114) -# ============================================================================ - - -def test_tensile_benchmark_client_missing_args(): - """PIN error path: TensileBenchmarkLibraryClient with no args exits(-1). - - Covers arg validation (lines 107-114): len(userArgs) < 2 check. - """ - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - - with pytest.raises(SystemExit) as exc_info: - M.TensileBenchmarkLibraryClient([]) - - assert exc_info.value.code == -1 - stderr_text = sys.stderr.getvalue() - assert "USAGE:" in stderr_text - finally: - sys.stderr = old_stderr - - -def test_tensile_benchmark_client_one_arg(): - """PIN error path: TensileBenchmarkLibraryClient with only CSV path exits(-1). - - Covers arg validation (lines 107-114): len(userArgs) < 2 check. - """ - with tempfile.TemporaryDirectory() as tmpdir: - csv_path = Path(tmpdir) / "sizes.csv" - csv_path.write_text("128,128,1,512\n") - - old_stderr = sys.stderr - try: - sys.stderr = io.StringIO() - - with pytest.raises(SystemExit) as exc_info: - M.TensileBenchmarkLibraryClient([str(csv_path)]) - - assert exc_info.value.code == -1 - stderr_text = sys.stderr.getvalue() - assert "USAGE:" in stderr_text - finally: - sys.stderr = old_stderr - - -# ============================================================================ -# Integration Test: CSV Parsing Paths (coverage lines 117-149) -# ============================================================================ - - -def test_tensile_benchmark_client_csv_parsing(mock_csv_file): - """PIN CSV parsing paths: TensileBenchmarkLibraryClient reads and processes CSV. - - Covers: - - Lines 117-120: problem sizes path parsing and output (realpath, write to stdout) - - Lines 122-129: library client command parsing and output (join, write to stdout) - - Lines 132-140: CSV file open, first row read, numIndices set - - Lines 141-149: header building (loop over indices, format header line) - - Lines 152-154: benchmark first row (calls BenchmarkProblemSize) - - Lines 157-159: loop over remaining CSV rows (calls BenchmarkProblemSize) - - This test drives the CSV parsing and benchmark-calling paths. It mocks - BenchmarkProblemSize to return synthetic data (avoiding subprocess issues). - The function will raise TypeError when PrintStats calls the broken median() - function (Python 2/3 division bug, already pinned in test_stats_char.py). - - We assert that the TypeError occurs AFTER the CSV parsing is exercised, - confirming the target lines were reached. - """ - from unittest.mock import patch - - # Create a mock BenchmarkProblemSize that returns valid lists without subprocess. - def mock_benchmark_fn(cmd, row): - # Return synthetic results matching the expected format. - return [500.0, 550.0, 480.0], [1.0, 0.95, 1.05] - - old_stdout = sys.stdout - old_stderr = sys.stderr - try: - sys.stdout = io.StringIO() - sys.stderr = io.StringIO() - - with patch("Tensile.TensileBenchmarkLibraryClient.BenchmarkProblemSize", mock_benchmark_fn): - # The function will reach CSV parsing (lines 117-159) and then fail - # on the median bug in PrintStats (line 163). This pins that the - # CSV and benchmark paths were exercised. - with pytest.raises(TypeError, match="list indices must be integers"): - user_args = [mock_csv_file, "/fake/client"] - M.TensileBenchmarkLibraryClient(user_args) - - stdout_text = sys.stdout.getvalue() - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - - # PIN: CSV header and path/command parsing succeeded before the median error. - # Both are written to stdout before the error. - assert "ProblemSizesPath:" in stdout_text - assert "LibraryClientCommand:" in stdout_text - # The header line is built and printed before PrintStats is called (which errors). - assert "size0," in stdout_text - - -def test_main_entry_point(mock_csv_file): - """PIN main() entry point: dispatches to TensileBenchmarkLibraryClient - with sys.argv[1:]. - - Covers line 180: TensileBenchmarkLibraryClient(sys.argv[1:]). - - The main() function is expected to fail with the same TypeError from the - median bug that TensileBenchmarkLibraryClient encounters. - """ - from unittest.mock import patch - - def mock_benchmark_fn(cmd, row): - # Return synthetic results matching the expected format. - return [500.0, 550.0, 480.0], [1.0, 0.95, 1.05] - - import sys as sys_module - - old_argv = sys_module.argv - old_stdout = sys.stdout - old_stderr = sys.stderr - try: - sys_module.argv = ["prog", mock_csv_file, "/fake/client"] - sys.stdout = io.StringIO() - sys.stderr = io.StringIO() - - with patch("Tensile.TensileBenchmarkLibraryClient.BenchmarkProblemSize", mock_benchmark_fn): - # main() delegates to TensileBenchmarkLibraryClient, which fails on median bug. - with pytest.raises(TypeError, match="list indices must be integers"): - M.main() - - stdout_text = sys.stdout.getvalue() - # PIN: main() reached TensileBenchmarkLibraryClient (line 180), which produced output - # before hitting the error. - assert "ProblemSizesPath:" in stdout_text - finally: - sys_module.argv = old_argv - sys.stdout = old_stdout - sys.stderr = old_stderr diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_stats_char.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_stats_char.py deleted file mode 100644 index 29bcf5be528f..000000000000 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/TensileBenchmarkLibraryClient/test_stats_char.py +++ /dev/null @@ -1,30 +0,0 @@ -################################################################################ -# Characterization tests for Tensile.TensileBenchmarkLibraryClient — stats. -# -# ADD-ONLY. The client driver (BenchmarkProblemSize subprocess, PrintStats, -# TensileBenchmarkLibraryClient) is resistance; this pins the pure stats helpers -# and the median() py3 bug. -################################################################################ -import importlib - -import pytest - -pytestmark = pytest.mark.unit - -M = importlib.import_module("Tensile.TensileBenchmarkLibraryClient") - - -def test_mean(): - assert M.mean([2, 4, 6]) == 4 - - -def test_stddev(): - # sample stddev of [2,4,6]: mean 4, sumsq 8, /(n-1)=4 -> 2.0 - assert M.stddev([2, 4, 6]) == 2.0 - - -def test_median_is_broken_in_py3(): - # LATENT BUG (pinned): median uses sortedList[len(sortedList)/2]; in py3 `/` - # yields a float, and a float is not a valid list index -> TypeError. - with pytest.raises(TypeError): - M.median([3, 1, 2]) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/coverage-baseline.json b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/coverage-baseline.json index 7200de5f4176..8c08ad8f517c 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/coverage-baseline.json +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/characterization/coverage-baseline.json @@ -109,7 +109,6 @@ "Tensile/Tensile.py": 80.0, "Tensile/TensileBenchmarkCluster.py": 99.51, "Tensile/TensileBenchmarkClusterScripts.py": 100.0, - "Tensile/TensileBenchmarkLibraryClient.py": 54.41, "Tensile/TensileCreateLibrary/ParseArguments.py": 100.0, "Tensile/TensileCreateLibrary/Run.py": 88.53, "Tensile/TensileCreateLibrary/__init__.py": 100.0,