Redesign plotting YAML format

TODO.md

@@ -4,7 +4,7 @@
   - [x] Ingest YAML of schema
   - [x] Generate sbatch files from YAML
 - [ ] Improvements
-  - [ ] Results analysis
+  - [x] Results analysis
     - [x] Result aggregation
     - [x] Result analysis
     - [ ] Statistical re-runs
@@ -14,12 +14,12 @@
     - [ ] Default groups for executables
     - [x] Optional key to disable test benches in YAML
   - [ ] Optional extras
-    - [ ] dry run flag
+    - [x] dry run flag
     - [x] Argument parsing from the command line
     - [ ] Unit tests
     - [ ] Documentations
     - [ ] Publish to PyPI
-    - [ ] Textual TUI
+    - [x] Textual TUI
       - [x] Original ideas
         - Visualise YAML schema structure
         - Spawn runs for benches/individual executables

src/hpc_multibench/test_bench.py

@@ -184,30 +184,31 @@ def report(self) -> None:
         }
 
         # Extract the outputs into the data format needed for the line plot
-        # TODO: Could pull out into analysis file?
-        data: dict[str, list[tuple[float, float]]] = {
-            run_name: [] for run_name in self.run_configuration_models
-        }
-        for run_configuration, output in run_outputs.values():
-            if output is not None:
-                metrics = self.extract_metrics(output)
-                if metrics is None:
-                    continue
-                data[run_configuration.name].append(
-                    (
-                        float(metrics[self.bench_model.analysis.plot.x]),
-                        float(metrics[self.bench_model.analysis.plot.y]),
+        for plot in self.bench_model.analysis.line_plots:
+            # TODO: Could extraction function out into analysis file?
+            data: dict[str, list[tuple[float, float]]] = {
+                run_name: [] for run_name in self.run_configuration_models
+            }
+            for run_configuration, output in run_outputs.values():
+                if output is not None:
+                    metrics = self.extract_metrics(output)
+                    if metrics is None:
+                        continue
+                    data[run_configuration.name].append(
+                        (
+                            float(metrics[plot.x]),
+                            float(metrics[plot.y]),
+                        )
                     )
-                )
 
-        for name, results in data.items():
-            print(name, results)
-            plt.plot(*zip(*results, strict=True), marker="x", label=name)
-        plt.xlabel(self.bench_model.analysis.plot.x)
-        plt.ylabel(self.bench_model.analysis.plot.y)
-        plt.title(self.bench_model.analysis.plot.title)
-        plt.legend()
-        plt.show()
+            for name, results in data.items():
+                print(name, results)
+                plt.plot(*zip(*results, strict=True), marker="x", label=name)
+            plt.xlabel(plot.x)
+            plt.ylabel(plot.y)
+            plt.title(plot.title)
+            plt.legend()
+            plt.show()
 
         # print("\n".join(str(x) for x in self.run_configurations_metadata))
         # Load mappings from run config/args to slurm job ids

src/hpc_multibench/yaml_model.py

@@ -48,21 +48,45 @@ def realise(
         return run
 
 
-class PlotModel(BaseModel):
-    """A Pydantic model for plotting two values against each other."""
+class LinePlotModel(BaseModel):
+    """A Pydantic model for a line plot of two variables."""
 
+    title: str
     x: str
     y: str
-    title: str = "Performance Plot"
 
-    # TODO: Needs work to expand capability
+
+# class RooflinePlotModel(BaseModel):
+#     """A Pydantic model for a roofline plot from two metrics."""
+
+#     title: str
+#     gflops_per_sec: str
+#     flops_per_byte: str
+
+
+# class BarChartModel(BaseModel):
+#     """A Pydantic model for a bar chart of a single variable."""
+
+#     title: str
+#     y: str
+
+
+# class ExportModel(BaseModel):
+#     """A Pydantic model for a exporting a set of metrics."""
+
+#     metrics: list[str]
+#     filename: Path | None = None
 
 
 class AnalysisModel(BaseModel):
     """A Pydantic model for a test bench's analysis operations."""
 
     metrics: dict[str, str]
-    plot: PlotModel
+    # TODO: Offer singular interface `line_plot` which is just one plot
+    line_plots: list[LinePlotModel] = []
+    # rooflines: list[RooflinePlotModel] = []
+    # bar_charts: list[BarChartModel] = []
+    # exports: list[BarChartModel] = []
 
 
 class BenchModel(BaseModel):

yaml_examples/example.yaml

@@ -17,7 +17,6 @@ run_configurations:
 
 benches:
   "strong-scaling":
-    enabled: True
     run_configurations:
       - "cpp-hybrid"
     matrix:
@@ -30,7 +29,45 @@ benches:
     analysis:
       metrics:
         "Mesh z size": "nz: (\\d+)"
+        "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}"
         "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary"
-      plot:
-        x: "Mesh z size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "Strong Scaling Plot"
+          x: "Mesh z size"
+          y: "Total time (s)"
+      # roofline_plots:
+      #   - title: "Strong Scaling Roofline"
+      #     gflops_per_sec: "Mesh z size"
+      #     flops_per_byte: "Total time (s)"
+      # bar_charts:
+      #   - title: "Strong Scaling Chart"
+      #     y: "Total time (s)"
+      # exports:
+      #   - filename: "./export.csv"
+      #     metrics:
+      #       - "Mesh z size"
+      #       - "Total time (s)"
+
+
+
+  "weak-scaling":
+    enabled: False
+    run_configurations:
+      - "cpp-hybrid"
+    matrix:
+      args:
+        - "64 64 64"
+      run_command:
+        - "mpirun -np 1 ./test_HPCCG"
+        - "mpirun -np 2 ./test_HPCCG"
+        - "mpirun -np 4 ./test_HPCCG"
+        - "mpirun -np 8 ./test_HPCCG"
+        - "mpirun -np 16 ./test_HPCCG"
+    analysis:
+      metrics:
+        "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}"
+        "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary"
+      line_plots:
+        - title: "Weak Scaling Plot"
+          x: "MPI Ranks"
+          y: "Total time (s)"

yaml_examples/kudu/cpp_kokkos_rust.yaml

@@ -123,6 +123,7 @@ benches:
         "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
         "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
         "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-      plot:
-        x: "Mesh x size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "C++, Kokkos, and Rust Version Comparison"
+          x: "Mesh x size"
+          y: "Total time (s)"

yaml_examples/kudu/original_versions.yaml

@@ -89,6 +89,7 @@ benches:
         "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
         "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
         "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-      plot:
-        x: "Mesh x size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "C++ Versions Comparison"
+          x: "Mesh x size"
+          y: "Total time (s)"

yaml_examples/kudu/parallelism_strategies.yaml

@@ -147,9 +147,10 @@ benches:
         "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
         "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
         "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-      plot:
-        x: "Mesh x size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "Reference Implementation Comparison"
+          x: "Mesh x size"
+          y: "Total time (s)"
 
   "parallel":
     run_configurations:
@@ -183,9 +184,10 @@ benches:
         "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
         "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
         "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-      plot:
-        x: "Mesh x size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "Parallel Implementation Comparison"
+          x: "Mesh x size"
+          y: "Total time (s)"
 
   # "mpi":
   #   run_configurations:
@@ -213,9 +215,10 @@ benches:
   #       "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
   #       "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
   #       "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-  #     plot:
-  #       x: "Mesh x size"
-  #       y: "Total time (s)"
+      # line_plots:
+      #   - title: "MPI Implementation Comparison"
+      #     x: "Mesh x size"
+      #     y: "Total time (s)"
 
   # "hybrid":
   #   run_configurations:
@@ -250,6 +253,7 @@ benches:
   #       "ddot mflops": "MFLOPS Summary:[\\s\\S]*DDOT\\s*: ([\\d\\.]+)"
   #       "waxpby mflops": "MFLOPS Summary:[\\s\\S]*WAXPBY\\s*: ([\\d\\.]+)"
   #       "sparsemv mflops": "MFLOPS Summary:[\\s\\S]*SPARSEMV\\s*: ([\\d\\.]+)"
-  #     plot:
-  #       x: "Mesh x size"
-  #       y: "Total time (s)"
+      # line_plots:
+      #   - title: "MPI & Parallel Implementation Comparison"
+      #     x: "Mesh x size"
+      #     y: "Total time (s)"

yaml_examples/kudu/strong_weak_scaling.yaml

@@ -49,10 +49,10 @@ benches:
         "Mesh z size": "nz: (\\d+)"
         "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}"
         "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary"
-      plot:
-        title: "Strong Scaling Plot"
-        x: "Mesh z size"
-        y: "Total time (s)"
+      line_plots:
+        - title: "Strong Scaling Plot"
+          x: "Mesh z size"
+          y: "Total time (s)"
 
   "weak-scaling":
     run_configurations:
@@ -73,7 +73,7 @@ benches:
       metrics:
         "MPI Ranks": "=== RUN INSTANTIATION ===\n\\{.*run_command: mpirun -np (\\d+).*\\}"
         "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary"
-      plot:
-        title: "Weak Scaling Plot"
-        x: "MPI Ranks"
-        y: "Total time (s)"
+      line_plots:
+        - title: "Weak Scaling Plot"
+          x: "MPI Ranks"
+          y: "Total time (s)"