From 8b756f22f1b120f77353f0b051c7f91043d0b9ff Mon Sep 17 00:00:00 2001 From: EdmundGoodman Date: Sun, 25 Feb 2024 03:50:04 +0000 Subject: [PATCH] Redesign plotting YAML format --- TODO.md | 6 +-- src/hpc_multibench/test_bench.py | 45 ++++++++++--------- src/hpc_multibench/yaml_model.py | 34 +++++++++++--- yaml_examples/example.yaml | 45 +++++++++++++++++-- yaml_examples/kudu/cpp_kokkos_rust.yaml | 7 +-- yaml_examples/kudu/original_versions.yaml | 7 +-- .../kudu/parallelism_strategies.yaml | 28 +++++++----- yaml_examples/kudu/strong_weak_scaling.yaml | 16 +++---- 8 files changed, 128 insertions(+), 60 deletions(-) diff --git a/TODO.md b/TODO.md index 12495e5..4cbf4db 100644 --- a/TODO.md +++ b/TODO.md @@ -4,7 +4,7 @@ - [x] Ingest YAML of schema - [x] Generate sbatch files from YAML - [ ] Improvements - - [ ] Results analysis + - [x] Results analysis - [x] Result aggregation - [x] Result analysis - [ ] Statistical re-runs @@ -14,12 +14,12 @@ - [ ] Default groups for executables - [x] Optional key to disable test benches in YAML - [ ] Optional extras - - [ ] dry run flag + - [x] dry run flag - [x] Argument parsing from the command line - [ ] Unit tests - [ ] Documentations - [ ] Publish to PyPI - - [ ] Textual TUI + - [x] Textual TUI - [x] Original ideas - Visualise YAML schema structure - Spawn runs for benches/individual executables diff --git a/src/hpc_multibench/test_bench.py b/src/hpc_multibench/test_bench.py index fda005e..20544ea 100755 --- a/src/hpc_multibench/test_bench.py +++ b/src/hpc_multibench/test_bench.py @@ -184,30 +184,31 @@ def report(self) -> None: } # Extract the outputs into the data format needed for the line plot - # TODO: Could pull out into analysis file? - data: dict[str, list[tuple[float, float]]] = { - run_name: [] for run_name in self.run_configuration_models - } - for run_configuration, output in run_outputs.values(): - if output is not None: - metrics = self.extract_metrics(output) - if metrics is None: - continue - data[run_configuration.name].append( - ( - float(metrics[self.bench_model.analysis.plot.x]), - float(metrics[self.bench_model.analysis.plot.y]), + for plot in self.bench_model.analysis.line_plots: + # TODO: Could extraction function out into analysis file? + data: dict[str, list[tuple[float, float]]] = { + run_name: [] for run_name in self.run_configuration_models + } + for run_configuration, output in run_outputs.values(): + if output is not None: + metrics = self.extract_metrics(output) + if metrics is None: + continue + data[run_configuration.name].append( + ( + float(metrics[plot.x]), + float(metrics[plot.y]), + ) ) - ) - for name, results in data.items(): - print(name, results) - plt.plot(*zip(*results, strict=True), marker="x", label=name) - plt.xlabel(self.bench_model.analysis.plot.x) - plt.ylabel(self.bench_model.analysis.plot.y) - plt.title(self.bench_model.analysis.plot.title) - plt.legend() - plt.show() + for name, results in data.items(): + print(name, results) + plt.plot(*zip(*results, strict=True), marker="x", label=name) + plt.xlabel(plot.x) + plt.ylabel(plot.y) + plt.title(plot.title) + plt.legend() + plt.show() # print("\n".join(str(x) for x in self.run_configurations_metadata)) # Load mappings from run config/args to slurm job ids diff --git a/src/hpc_multibench/yaml_model.py b/src/hpc_multibench/yaml_model.py index 4f1b880..7180f48 100755 --- a/src/hpc_multibench/yaml_model.py +++ b/src/hpc_multibench/yaml_model.py @@ -48,21 +48,45 @@ def realise( return run -class PlotModel(BaseModel): - """A Pydantic model for plotting two values against each other.""" +class LinePlotModel(BaseModel): + """A Pydantic model for a line plot of two variables.""" + title: str x: str y: str - title: str = "Performance Plot" - # TODO: Needs work to expand capability + +# class RooflinePlotModel(BaseModel): +# """A Pydantic model for a roofline plot from two metrics.""" + +# title: str +# gflops_per_sec: str +# flops_per_byte: str + + +# class BarChartModel(BaseModel): +# """A Pydantic model for a bar chart of a single variable.""" + +# title: str +# y: str + + +# class ExportModel(BaseModel): +# """A Pydantic model for a exporting a set of metrics.""" + +# metrics: list[str] +# filename: Path | None = None class AnalysisModel(BaseModel): """A Pydantic model for a test bench's analysis operations.""" metrics: dict[str, str] - plot: PlotModel + # TODO: Offer singular interface `line_plot` which is just one plot + line_plots: list[LinePlotModel] = [] + # rooflines: list[RooflinePlotModel] = [] + # bar_charts: list[BarChartModel] = [] + # exports: list[BarChartModel] = [] class BenchModel(BaseModel): diff --git a/yaml_examples/example.yaml b/yaml_examples/example.yaml index fe20839..6379a6a 100644 --- a/yaml_examples/example.yaml +++ b/yaml_examples/example.yaml @@ -17,7 +17,6 @@ run_configurations: benches: "strong-scaling": - enabled: True run_configurations: - "cpp-hybrid" matrix: @@ -30,7 +29,45 @@ benches: analysis: metrics: "Mesh z size": "nz: (\\d+)" + "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}" "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary" - plot: - x: "Mesh z size" - y: "Total time (s)" + line_plots: + - title: "Strong Scaling Plot" + x: "Mesh z size" + y: "Total time (s)" + # roofline_plots: + # - title: "Strong Scaling Roofline" + # gflops_per_sec: "Mesh z size" + # flops_per_byte: "Total time (s)" + # bar_charts: + # - title: "Strong Scaling Chart" + # y: "Total time (s)" + # exports: + # - filename: "./export.csv" + # metrics: + # - "Mesh z size" + # - "Total time (s)" + + + + "weak-scaling": + enabled: False + run_configurations: + - "cpp-hybrid" + matrix: + args: + - "64 64 64" + run_command: + - "mpirun -np 1 ./test_HPCCG" + - "mpirun -np 2 ./test_HPCCG" + - "mpirun -np 4 ./test_HPCCG" + - "mpirun -np 8 ./test_HPCCG" + - "mpirun -np 16 ./test_HPCCG" + analysis: + metrics: + "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}" + "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary" + line_plots: + - title: "Weak Scaling Plot" + x: "MPI Ranks" + y: "Total time (s)" diff --git a/yaml_examples/kudu/cpp_kokkos_rust.yaml b/yaml_examples/kudu/cpp_kokkos_rust.yaml index 46ee61f..9a5084e 100644 --- a/yaml_examples/kudu/cpp_kokkos_rust.yaml +++ b/yaml_examples/kudu/cpp_kokkos_rust.yaml @@ -123,6 +123,7 @@ benches: "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - plot: - x: "Mesh x size" - y: "Total time (s)" + line_plots: + - title: "C++, Kokkos, and Rust Version Comparison" + x: "Mesh x size" + y: "Total time (s)" diff --git a/yaml_examples/kudu/original_versions.yaml b/yaml_examples/kudu/original_versions.yaml index f273893..3dddc4b 100644 --- a/yaml_examples/kudu/original_versions.yaml +++ b/yaml_examples/kudu/original_versions.yaml @@ -89,6 +89,7 @@ benches: "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - plot: - x: "Mesh x size" - y: "Total time (s)" + line_plots: + - title: "C++ Versions Comparison" + x: "Mesh x size" + y: "Total time (s)" diff --git a/yaml_examples/kudu/parallelism_strategies.yaml b/yaml_examples/kudu/parallelism_strategies.yaml index 405cc97..ed65745 100644 --- a/yaml_examples/kudu/parallelism_strategies.yaml +++ b/yaml_examples/kudu/parallelism_strategies.yaml @@ -147,9 +147,10 @@ benches: "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - plot: - x: "Mesh x size" - y: "Total time (s)" + line_plots: + - title: "Reference Implementation Comparison" + x: "Mesh x size" + y: "Total time (s)" "parallel": run_configurations: @@ -183,9 +184,10 @@ benches: "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - plot: - x: "Mesh x size" - y: "Total time (s)" + line_plots: + - title: "Parallel Implementation Comparison" + x: "Mesh x size" + y: "Total time (s)" # "mpi": # run_configurations: @@ -213,9 +215,10 @@ benches: # "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" # "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" # "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - # plot: - # x: "Mesh x size" - # y: "Total time (s)" + # line_plots: + # - title: "MPI Implementation Comparison" + # x: "Mesh x size" + # y: "Total time (s)" # "hybrid": # run_configurations: @@ -250,6 +253,7 @@ benches: # "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)" # "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)" # "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)" - # plot: - # x: "Mesh x size" - # y: "Total time (s)" + # line_plots: + # - title: "MPI & Parallel Implementation Comparison" + # x: "Mesh x size" + # y: "Total time (s)" diff --git a/yaml_examples/kudu/strong_weak_scaling.yaml b/yaml_examples/kudu/strong_weak_scaling.yaml index 19122b4..bfcbec0 100644 --- a/yaml_examples/kudu/strong_weak_scaling.yaml +++ b/yaml_examples/kudu/strong_weak_scaling.yaml @@ -49,10 +49,10 @@ benches: "Mesh z size": "nz: (\\d+)" "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}" "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary" - plot: - title: "Strong Scaling Plot" - x: "Mesh z size" - y: "Total time (s)" + line_plots: + - title: "Strong Scaling Plot" + x: "Mesh z size" + y: "Total time (s)" "weak-scaling": run_configurations: @@ -73,7 +73,7 @@ benches: metrics: "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}" "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary" - plot: - title: "Weak Scaling Plot" - x: "MPI Ranks" - y: "Total time (s)" + line_plots: + - title: "Weak Scaling Plot" + x: "MPI Ranks" + y: "Total time (s)"