Add more debugging and exclusive node use to try ot get MPI working r…

…eliablyt
EdmundGoodman · Feb 23, 2024 · 2b9eaf2 · 2b9eaf2
1 parent f19c3a3
commit 2b9eaf2
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 10 deletions.
diff --git a/src/hpc_multibench/configuration.py b/src/hpc_multibench/configuration.py
@@ -52,7 +52,12 @@ def sbatch_contents(self) -> str:
             sbatch_file += f"export {key}={value}\n"
 
         sbatch_file += "\necho '===== ENVIRONMENT ====='\n"
+        sbatch_file += "echo '=== CPU ARCHITECTURE ==='\n"
         sbatch_file += "lscpu\n"
+        sbatch_file += "echo '=== HOSTNAME ==='\n"
+        sbatch_file += "hostname\n"
+        sbatch_file += "echo '=== SLURM CONFIG ==='\n"
+        sbatch_file += "scontrol show job $SLURM_JOB_ID\n"
         sbatch_file += "echo\n"
 
         sbatch_file += "\necho '===== BUILD ====='\n"

diff --git a/yaml_examples/kudu/cpp_kokkos_rust.yaml b/yaml_examples/kudu/cpp_kokkos_rust.yaml
@@ -4,6 +4,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -18,6 +19,7 @@ run_configurations:
       "nodes": 2
       "ntasks-per-node": 4
       "cpus-per-task": 2
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads:
       - "cs402-mpi"
@@ -33,6 +35,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -47,6 +50,7 @@ run_configurations:
       "nodes": 2
       "ntasks-per-node": 4
       "cpus-per-task": 2
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads:
       - "cs402-mpi"
@@ -62,6 +66,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -76,6 +81,7 @@ run_configurations:
       "nodes": 2
       "ntasks-per-node": 4
       "cpus-per-task": 2
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads:
       - "cs402-mpi"

diff --git a/yaml_examples/kudu/original_versions.yaml b/yaml_examples/kudu/original_versions.yaml
@@ -3,7 +3,8 @@ run_configurations:
     sbatch_config:
       "nodes": 1
       "ntasks-per-node": 1
-      "cpus-per-task": 1
+      "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables: {}
@@ -17,6 +18,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -28,23 +30,25 @@ run_configurations:
 
   "cpp-mpi":
     sbatch_config:
-      "nodes": 2
-      "ntasks-per-node": 8
-      "cpus-per-task": 1
+      "nodes": 1
+      "ntasks-per-node": 16
+      "cpus-per-task": 2
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads:
       - "cs402-mpi"
     environment_variables: {}
     directory: "../0_cpp_versions/2_mpi"
     build_commands:
       - "make -j 8"
-    run_command: "mpirun -n 16 ./test_HPCCG"  # TODO: Select best MPI config
+    run_command: "mpirun -np 16 ./test_HPCCG"  # TODO: Select best MPI config
 
   "cpp-hybrid":
     sbatch_config:
-      "nodes": 2
-      "ntasks-per-node": 4
+      "nodes": 1
+      "ntasks-per-node": 16
       "cpus-per-task": 2
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads:
       - "cs402-mpi"
@@ -53,7 +57,7 @@ run_configurations:
     directory: "../0_cpp_versions/3_hybrid"
     build_commands:
       - "make -j 8"
-    run_command: "mpirun -n 16 ./test_HPCCG"  # TODO: Select best MPI config
+    run_command: "mpirun -np 16 ./test_HPCCG"  # TODO: Select best MPI config
 
 
 benches:
@@ -66,8 +70,8 @@ benches:
     matrix:
       args:
         - "100 100 100"
-        - "300 300 300"
-        - "500 500 500"
+        # - "300 300 300"
+        # - "500 500 500"
     analysis:
       metrics:
         "Mesh x size": "nx: (\\d+)"

diff --git a/yaml_examples/kudu/parallelism_strategies.yaml b/yaml_examples/kudu/parallelism_strategies.yaml
@@ -4,6 +4,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 1
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables: {}
@@ -17,6 +18,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -31,6 +33,7 @@ run_configurations:
   #     "nodes": 2
   #     "ntasks-per-node": 8
   #     "cpus-per-task": 1
+  #     "exclusive": "mcs"
   #     "mem-per-cpu": 1500
   #   module_loads:
   #     - "cs402-mpi"
@@ -45,6 +48,7 @@ run_configurations:
   #     "nodes": 2
   #     "ntasks-per-node": 4
   #     "cpus-per-task": 2
+  #     "exclusive": "mcs"
   #     "mem-per-cpu": 1500
   #   module_loads:
   #     - "cs402-mpi"
@@ -60,6 +64,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 1
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables: {}
@@ -73,6 +78,7 @@ run_configurations:
       "nodes": 1
       "ntasks-per-node": 1
       "cpus-per-task": 16
+      "exclusive": "mcs"
       "mem-per-cpu": 1500
     module_loads: []
     environment_variables:
@@ -87,6 +93,7 @@ run_configurations:
   #     "nodes": 2
   #     "ntasks-per-node": 8
   #     "cpus-per-task": 1
+  #     "exclusive": "mcs"
   #     "mem-per-cpu": 1500
   #   module_loads:
   #     - "cs402-mpi"
@@ -101,6 +108,7 @@ run_configurations:
   #     "nodes": 2
   #     "ntasks-per-node": 4
   #     "cpus-per-task": 2
+  #     "exclusive": "mcs"
   #     "mem-per-cpu": 1500
   #   module_loads:
   #     - "cs402-mpi"