Get mpi and hybrid configuration sweeps working

EdmundGoodman · Feb 25, 2024 · 0df6c4d · 0df6c4d
1 parent 5a402a3
commit 0df6c4d
Show file tree

Hide file tree

Showing 2 changed files with 89 additions and 34 deletions.
diff --git a/src/hpc_multibench/test_bench.py b/src/hpc_multibench/test_bench.py
@@ -176,6 +176,10 @@ def record(self, args: Namespace) -> None:
 
         # TODO: Need to account for case where build commands is in the
         # matrix, then just needs to be a long chain of dependencies
+        # TODO: Could spawn an extra job to just build then not run, which
+        # everything else depends on? Probably better to document that fastest
+        # job should be first, avoids spawning extraneous jobs which could all
+        # fail if the configuration is wrong...
 
         # Run all run configurations and store their slurm job ids
         run_configuration_job_ids: dict[RunConfiguration, int | None] = {}

diff --git a/yaml_examples/kudu/simple_rust.yaml b/yaml_examples/kudu/simple_rust.yaml
@@ -13,27 +13,53 @@ run_configurations:
       - "make -j 8"
     run_command: "./test_HPCCG"
 
+  "rust-reference":
+    sbatch_config:
+      "nodes": 1
+      "ntasks-per-node": 1
+      "cpus-per-task": 1
+      "exclusive": "mcs"
+      "mem": 60000  # Rust seems to use more memory, this heavily bottlenecks if too small
+    module_loads: []
+    environment_variables: {}
+    directory: "../5_iterators"
+    build_commands:
+      - "cargo build --release"
+    run_command: "./target/release/hpccg-rs"
+
   "cpp-openmp":
     sbatch_config:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 32
       "exclusive": "mcs"
-      "mem-per-cpu": 1875
+      "mem": 60000
     module_loads: []
     environment_variables: {}
     directory: "../0_cpp_versions/1_openmp"
     build_commands:
       - "make -j 8"
     run_command: "./test_HPCCG"
 
+  "rust-rayon":
+    sbatch_config:
+      "nodes": 1
+      "ntasks-per-node": 1
+      "cpus-per-task": 32
+      "exclusive": "mcs"
+      "mem": 60000
+    module_loads: []
+    environment_variables: {}
+    directory: "../6_parallel"
+    build_commands:
+      - "cargo build --release"
+    run_command: "./target/release/hpccg-rs"
+
   "cpp-mpi":
     sbatch_config:
-      "nodes": 2
-      "ntasks-per-node": 8
       "cpus-per-task": 1
       "exclusive": "mcs"
-      "mem-per-cpu": 1875
+      "mem": 60000
     module_loads:
       - "cs402-mpi"
     environment_variables: {}
@@ -42,45 +68,41 @@ run_configurations:
       - "make -j 8"
     run_command: "mpirun ./test_HPCCG"
 
-  "rust-reference":
+  "rust-mpi":
     sbatch_config:
-      "nodes": 1
-      "ntasks-per-node": 1
       "cpus-per-task": 1
       "exclusive": "mcs"
-      "mem": 60000  # Rust seems to use more memory, this heavily bottlenecks if too small
-    module_loads: []
+      "mem": 60000
+    module_loads:
+      - "cs402-mpi"
     environment_variables: {}
-    directory: "../5_iterators"
+    directory: "../7_mpi"
     build_commands:
       - "cargo build --release"
-    run_command: "./target/release/hpccg-rs"
+    run_command: "mpirun ./target/release/hpccg-rs"
 
-  "rust-rayon":
+  "cpp-hybrid":
     sbatch_config:
       "nodes": 1
-      "ntasks-per-node": 1
-      "cpus-per-task": 32
       "exclusive": "mcs"
-      "mem-per-cpu": 1875
-    module_loads: []
+      "mem": 60000
+    module_loads:
+      - "cs402-mpi"
     environment_variables: {}
-    directory: "../6_parallel"
+    directory: "../0_cpp_versions/2_mpi"
     build_commands:
-      - "cargo build --release"
-    run_command: "./target/release/hpccg-rs"
+      - "make -j 8"
+    run_command: "mpirun ./test_HPCCG"
 
-  "rust-mpi":
+  "rust-hybrid":
     sbatch_config:
-      "nodes": 2
-      "ntasks-per-node": 8
-      "cpus-per-task": 1
+      "nodes": 1
       "exclusive": "mcs"
-      "mem-per-cpu": 1875
+      "mem": 60000
     module_loads:
       - "cs402-mpi"
     environment_variables: {}
-    directory: "../7_mpi"
+    directory: "../8_hybrid"
     build_commands:
       - "cargo build --release"
     run_command: "mpirun ./target/release/hpccg-rs"
@@ -143,21 +165,50 @@ benches:
           x: "Mesh x size"
           y: "Wall time (s)"
 
-
-  "mpi-single-node-config-sweep":
+  "mpi-config-sweep":
+    enabled: False
     run_configurations:
       - "cpp-mpi"
       - "rust-mpi"
     matrix:
       args:
         - "100 100 100"
-      # [sbatch_config, environment_variables]:
-      #   - [{"ntasks-per-node": 1}, {"OMP_NUM_THREADS": 40}]  ## TODO: Figure out best for one MPI node, then duplicate across many
-      #   - [{"ntasks-per-node": 2}, {"OMP_NUM_THREADS": 20}]
-      #   - [{"ntasks-per-node": 4}, {"OMP_NUM_THREADS": 10}]
-      #   - [{"ntasks-per-node": 10}, {"OMP_NUM_THREADS": 4}]
-      #   - [{"ntasks-per-node": 20}, {"OMP_NUM_THREADS": 2}]
-      #   - [{"ntasks-per-node": 40}, {"OMP_NUM_THREADS": 1}]
+      sbatch_config:
+        - {"nodes": 1, "ntasks-per-node": 1}
+        - {"nodes": 1, "ntasks-per-node": 2}
+        - {"nodes": 1, "ntasks-per-node": 4}
+        - {"nodes": 1, "ntasks-per-node": 8}
+        - {"nodes": 2, "ntasks-per-node": 1}
+        - {"nodes": 2, "ntasks-per-node": 2}
+        - {"nodes": 2, "ntasks-per-node": 4}
+    analysis:
+      metrics:
+        "Mesh x size": "nx: (\\d+)"
+        "Mesh y size": "ny: (\\d+)"
+        "Mesh z size": "nz: (\\d+)"
+        "Total time (s)": "Time Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nFLOPS Summary"
+        "Total flops": "FLOPS Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)[\\s\\S]*\nMFLOPS Summary"
+        "Total mflops": "MFLOPS Summary:[\\s\\S]*Total\\s*: ([\\d\\.]+)"
+        "Wall time (s)": "real\\s([\\d\\.]+)\nuser"
+      line_plots:
+        - title: "MPI Configuration Sweep"
+          x: "Mesh x size"
+          y: "Wall time (s)"
+
+  "hybrid-single-node-config-sweep":
+    run_configurations:
+      - "cpp-hybrid"
+      - "rust-hybrid"
+    matrix:
+      args:
+        - "100 100 100"
+      [sbatch_config, environment_variables]:
+        - [{"ntasks-per-node": 1, "cpus-per-task": 40}, {"OMP_NUM_THREADS": 40, "RAYON_NUM_THREADS": 40}]
+        - [{"ntasks-per-node": 2, "cpus-per-task": 20}, {"OMP_NUM_THREADS": 20, "RAYON_NUM_THREADS": 20}]
+        - [{"ntasks-per-node": 4, "cpus-per-task": 10}, {"OMP_NUM_THREADS": 10, "RAYON_NUM_THREADS": 10}]
+        - [{"ntasks-per-node": 10, "cpus-per-task": 4}, {"OMP_NUM_THREADS": 4, "RAYON_NUM_THREADS": 4}]
+        - [{"ntasks-per-node": 20, "cpus-per-task": 2}, {"OMP_NUM_THREADS": 2, "RAYON_NUM_THREADS": 2}]
+        - [{"ntasks-per-node": 40, "cpus-per-task": 1}, {"OMP_NUM_THREADS": 1, "RAYON_NUM_THREADS": 1}]
     analysis:
       metrics:
         "Mesh x size": "nx: (\\d+)"